diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index ad749799..6adf9bbc 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -7,7 +7,7 @@ **Ruby Version:** -**Framework Version (RSpec, Minitest, FactoryGirl, Rails, whatever):** +**Framework Version (RSpec, Minitest, FactoryBot, Rails, whatever):** **TestProf Version:** diff --git a/.github/workflows/docs-lint.yml b/.github/workflows/docs-lint.yml index 716b4235..571ae446 100644 --- a/.github/workflows/docs-lint.yml +++ b/.github/workflows/docs-lint.yml @@ -6,65 +6,14 @@ on: - master paths: - "**/*.md" + - ".github/workflows/docs-lint.yml" pull_request: paths: - "**/*.md" + - ".github/workflows/docs-lint.yml" jobs: - markdownlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Run Markdown linter - run: | - gem install mdl - mdl docs - rubocop: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Lint Markdown files with RuboCop - run: | - gem install bundler - bundle install --gemfile gemfiles/rubocop.gemfile --jobs 4 --retry 3 - bundle exec --gemfile gemfiles/rubocop.gemfile rubocop -c .rubocop-md.yml - forspell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install Hunspell - run: | - sudo apt-get install hunspell - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Cache installed gems - uses: actions/cache@v1 - with: - path: /home/runner/.rubies/ruby-2.7.0/lib/ruby/gems/2.7.0 - key: gems-cache-${{ runner.os }} - - name: Install Forspell - run: gem install forspell - - name: Run Forspell - run: forspell docs/ - liche: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v1 - with: - go-version: 1.13.x - - name: Run liche - env: - GO111MODULE: "on" - run: | - export PATH=$PATH:$(go env GOPATH)/bin - go get -u github.com/raviqqe/liche - liche -r docs -d docs || true + docs-lint: + uses: anycable/github-actions/.github/workflows/docs-lint.yml@master + with: + lychee-args: docs/* -v README.md CHANGELOG.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..f1d5b7bf --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Release gems +on: + workflow_dispatch: + push: + tags: + - v* + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch current tag as annotated. See https://github.com/actions/checkout/issues/290 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.2 + - name: Configure RubyGems Credentials + uses: rubygems/configure-rubygems-credentials@main + - name: Publish to RubyGems + run: | + gem install gem-release + gem release diff --git a/.github/workflows/rspec-jruby.yml b/.github/workflows/rspec-jruby.yml index 98df8299..e0df99c3 100644 --- a/.github/workflows/rspec-jruby.yml +++ b/.github/workflows/rspec-jruby.yml @@ -9,26 +9,18 @@ on: jobs: rspec-jruby: runs-on: ubuntu-latest + timeout-minutes: 30 env: BUNDLE_JOBS: 4 BUNDLE_RETRY: 3 + BUNDLE_GEMFILE: gemfiles/jruby.gemfile + LET_IT_BE_IVAR_PREFIX: "@_matroskin_" steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: /home/runner/bundle - key: bundle-${{ hashFiles('**/gemfiles/jruby.gemfile') }}-${{ hashFiles('**/*.gemspec') }} - restore-keys: | - bundle- + - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: - ruby-version: jruby-9.2.15.0 - - name: Bundle install - run: | - bundle config path /home/runner/bundle - bundle config --global gemfile gemfiles/jruby.gemfile - bundle install - bundle update + ruby-version: jruby + bundler-cache: true - name: Run RSpec run: | bundle exec rspec --force-color diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 398e87de..075e1054 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -9,34 +9,69 @@ on: jobs: rspec: runs-on: ubuntu-latest + timeout-minutes: 10 env: BUNDLE_JOBS: 4 BUNDLE_RETRY: 3 + BUNDLE_GEMFILE: ${{ matrix.gemfile }} + CI: true + POSTGRES_URL: postgres://postgres:postgres@localhost:5432 + MYSQL_URL: mysql2://rails:rails@127.0.0.1:3306 + DB: ${{ matrix.db }} + MULTI_DB: ${{ matrix.multi_db }} + # Use postgres for all DB to avoid dealing with PG db creation + DB_NAME: postgres strategy: fail-fast: false matrix: - ruby: [2.7] - gemfile: ["gemfiles/activerecord6.gemfile"] + ruby: ["3.3"] + gemfile: ["gemfiles/activerecord8.gemfile"] db: ["postgres"] include: - - ruby: 3.0 - gemfile: "gemfiles/activerecord6.gemfile" - db: "sqlite" - - ruby: 3.0 + - ruby: "4.0" + gemfile: "gemfiles/activerecord8.gemfile" + db: "postgres" + multi_db: "true" + - ruby: "4.0" + gemfile: "gemfiles/railsmaster.gemfile" + db: "mysql" + - ruby: "4.0" gemfile: "gemfiles/railsmaster.gemfile" db: "sqlite" - - ruby: 2.7 - gemfile: "gemfiles/activerecord60.gemfile" + - ruby: "3.4" + gemfile: "gemfiles/activerecord8.gemfile" db: "postgres" - - ruby: 2.6 - gemfile: "gemfiles/activerecord5.gemfile" + multi_db: "true" + - ruby: "3.4" + gemfile: "gemfiles/railsmaster.gemfile" + db: "mysql" + - ruby: "3.3" + gemfile: "Gemfile" + db: "sqlite" + - ruby: "3.3" + gemfile: "gemfiles/activerecord72.gemfile" + db: "sqlite-file" + - ruby: "3.2" + gemfile: "gemfiles/activerecord7.gemfile" db: "sqlite-file" - - ruby: 2.5 - gemfile: "gemfiles/activerecord5.gemfile" + multi_db: "true" + - ruby: "3.1" + gemfile: "gemfiles/activerecord7.gemfile" + db: "postgres" + multi_db: "true" + - ruby: "3.0" + gemfile: "gemfiles/activerecord6.gemfile" + db: "sqlite" + db_url: ~ + - ruby: "3.0" + gemfile: "gemfiles/rspecrails4.gemfile" + db: "sqlite" + - ruby: "3.4" + gemfile: "gemfiles/railsmaster.gemfile" db: "sqlite" services: postgres: - image: postgres:12 + image: postgres:latest ports: ["5432:5432"] env: POSTGRES_PASSWORD: postgres @@ -45,32 +80,29 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + mysql: + image: mysql:8 + ports: ["3306:3306"] + env: + MYSQL_PASSWORD: rails + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: postgres + MYSQL_USER: rails + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: /home/runner/bundle - key: bundle-${{ matrix.ruby }}-${{ matrix.gemfile }}-${{ hashFiles(matrix.gemfile) }}-${{ hashFiles('**/*.gemspec') }} - restore-keys: | - bundle-${{ matrix.ruby }}-${{ matrix.gemfile }}- - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} + - uses: actions/checkout@v3 - name: Install system deps run: | sudo apt-get update - sudo apt-get install libsqlite3-dev libpq-dev postgresql-client-12 - - name: Bundle install - run: | - bundle config path /home/runner/bundle - bundle config --global gemfile ${{ matrix.gemfile }} - bundle install - bundle update + sudo apt-get install libsqlite3-dev libpq-dev postgresql-client + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true - name: Run RSpec - env: - DB: ${{ matrix.db }} - DATABASE_URL: postgres://postgres:postgres@localhost:5432 - DB_NAME: postgres run: | bundle exec rspec --force-color - diff --git a/.github/workflows/rubocop.yml b/.github/workflows/rubocop.yml index 11dc3708..5c9102c6 100644 --- a/.github/workflows/rubocop.yml +++ b/.github/workflows/rubocop.yml @@ -9,12 +9,16 @@ on: jobs: rubocop: runs-on: ubuntu-latest + env: + BUNDLE_JOBS: 4 + BUNDLE_RETRY: 3 + BUNDLE_GEMFILE: "gemfiles/rubocop.gemfile" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.7 + ruby-version: 3.2 + bundler-cache: true - name: Lint Ruby code with RuboCop run: | - bundle install --gemfile gemfiles/rubocop.gemfile --jobs 4 --retry 3 - bundle exec --gemfile gemfiles/rubocop.gemfile rubocop + bundle exec rubocop diff --git a/.gitignore b/.gitignore index be6fa8bb..20d4d8a4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,5 @@ gemfiles/*.lock spec/integrations/fixtures/rspec/tmp/ spec/integrations/fixtures/rspec/gemfiles/*.lock -Gemfile.local +Gemfile.local* tmp/ diff --git a/.mdlrc b/.mdlrc index 094d3e0d..99fa594c 100644 --- a/.mdlrc +++ b/.mdlrc @@ -1 +1 @@ -rules "~MD013", "~MD033" +rules "~MD013", "~MD033", "~MD007", "~MD032" diff --git a/.rubocop-md.yml b/.rubocop-md.yml index 301f4b00..10559438 100644 --- a/.rubocop-md.yml +++ b/.rubocop-md.yml @@ -1,9 +1,8 @@ inherit_from: ".rubocop.yml" -require: +plugins: - rubocop-md - AllCops: Include: - '**/*.md' diff --git a/.rubocop.yml b/.rubocop.yml index 2578d186..7e947b63 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -13,7 +13,7 @@ AllCops: - 'gemfiles/**/*' DisplayCopNames: true SuggestExtensions: false - TargetRubyVersion: 2.5 + TargetRubyVersion: 2.7 Standard/BlockSingleLineBraces: Enabled: false @@ -30,10 +30,9 @@ Style/TrailingCommaInHashLiteral: Layout/ParameterAlignment: EnforcedStyle: with_first_parameter -# See https://github.com/rubocop-hq/rubocop/issues/4222 -Lint/AmbiguousBlockAssociation: - Exclude: - - 'spec/**/*' +# See https://github.com/rubocop/rubocop/issues/11521 +Lint/FormatParameterMismatch: + Enabled: false Naming/FileName: Exclude: diff --git a/BACKERS.md b/BACKERS.md index 3ee5b600..25793878 100644 --- a/BACKERS.md +++ b/BACKERS.md @@ -1,5 +1,13 @@ # Sponsors & Backers +## Companies + +- [GitHub](https://github.com/github) +- [Pennylane](https://github.com/pennylane-hq) + ## Personal Sponsors -Be the first one :) +- [Makar Ermokhin](https://github.com/Earendil95) +- [Burkhard Vogel-Kreykenbohm](https://github.com/bvogel) +- [Charles Sistovaris](https://github.com/charly) +- [@Jwaterhouse052](https://github.com/Jwaterhouse052) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8840f3ef..e5ca1e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,228 @@ # Change log -## master (unrealeased) +## master (unreleased) + +## 1.5.2 (2026-02-03) + +- Avoid using `Gem.loaded_specs` methods in RuboCop plugin version check. ([@Rylan12][]) + +## 1.5.1 (2026-01-27) + +- Fix RuboCop plugin. ([@palkan][]) + +Now you should use `--plugin test-prof` or `plugins: [test-prof]` (in YAML) (so LintRoller can correctly obtain the plugin class name from the gemspec). + +## 1.5.0 (2025-12-04) + +- Logging: support Rails 8.2 structured events based logging. ([@palkan][]) + +- Allow using AnyFixture DSL through module inclusion, not refinements. ([@palkan][]) + +In Rails 7.2+, refined `#fixture` no longer works since there is a same-called method. So, from now on we recommend including the DSL module, instead of _using_ it. + +## 1.4.5. (2025-05-09) πŸŽ‡ + +- FactoryProf: Add truncate_names configuration parameter. ([@skaestle][]) + +- Update Rubocop setup to support new plugins system. ([@julianpasquale]) + +Now you can truncate long factory-names when using the simple output mode. + +Set `FPROF_TRUNCATE_NAMES=1` env var or set it through `FactoryProf` configuration: + +```ruby +TestProf::FactoryProf.configure do |config| + config.truncate_names = true +end +``` + +## 1.4.4 (2025-01-03) + +- Fix _stamping_ specs with single quotes with RSpec Stamp. ([@elasticspoon][]) + +## 1.4.3 (2024-12-18) + +- Fix handling new (lazy) connection pools in `before_all`. ([@palkan][]) + +- Updates Rubocop::Cop code to comply with most modern API. ([@aseroff][]) + +## 1.4.2 (2024-09-03) πŸ—“οΈ + +- Ignore default scopes in `ActiveRecord::Base#refind`. ([@palkan][]) + +## 1.4.1 (2024-08-23) + +- Skips loading the ActiveRecord adapter for runs where RSpec --dry-run mode is enabled. ([@devinburnette][]) + +## 1.4.0 (2024-08-12) + +- AnyFixture: Disable fixtures cache within _clean fixture_ context. Automatically _refind_ records when using `#fixture`. ([@palkan][]) + +- Add new TPS (tests per second) profiler. ([@palkan][]) + +- FactoryDefault: add Fabrication support. ([@palkan][]) + +- Drop support for **Ruby <2.7** and **Rails <6**. + +- FactoryDefault: Add `#get_factory_default`. [[@john-h-k][]] + +- Add variations information to FactorProf reports. ([@lHydra][]) + +Get info on traits/overrides used by running `FPROF=1 FPROF_VARS=1 `. + +- Add support for `report_duplicates` config option for `let_it_be` ([@lHydra][]) + +- Support latest Timecop patching `Process.clock_gettime`. ([@palkan][]) + +- Vernier: Add hooks configuration parameter. ([@lHydra][]) + +Now you can add more insights to the resulting report by adding event markers from Active Support Notifications. +To do this, specify the `TEST_VERNIER_HOOKS=rails` env var or set it through `Vernier` configuration: + +```ruby +TestProf::Vernier.configure do |config| + config.hooks = :rails +end +``` + +- FactoryProf: Add threshold configuration parameter. ([@lHydra][]) + +Now you can ignore factories which total number of calls is less than the provided threshold. To do this, specify +the `FPROF_THRESHOLD=30` env var or set it through `FactoryProf` configuration: + +```ruby +TestProf::FactoryProf.configure do |config| + config.threshold = 30 +end +``` + +## 1.3.3 (2024-04-19) + +- Fix MemProf bugs. ([@palkan][]) + +## 1.3.2 (2024-03-08) 🌷 + +- Add Minitest support for TagProf. ([@lioneldebauge][]) + +## 1.3.1 (2023-12-12) + +- Add support for dumping FactoryProf results in JSON format. ([@uzushino][]) + +## 1.3.0 (2023-11-21) + +- Add Vernier integration. ([@palkan][]) + +- StackProf uses JSON format by default. ([@palkan][]) + +- MemoryProf ia added. ([@Vankiru][]) + +## 1.2.3 (2023-09-11) + +- Minor fixes and dependencies upgrades. + +## 1.2.2 (2023-06-27) + +- Ignore inaccessible connection pools in `before_all`. ([@bf4][]) + +See [#267](https://github.com/test-prof/test-prof/pull/267). + +## 1.2.1 (2023-03-22) + +- Fix regression with `before_all(setup_fixtures: true)` and `rspec-rails` v6.0+. ([@palkan][]) + +- Upgrade to RubyProf 1.4+. ([@palkan][]) + +## 1.2.0 (2023-02-07) + +- Add support for multiple databases to `before_all` / `let_it_be` with Active Record. ([@rutgerw][]) + +## 1.1.0 (2022-12-06) + +- LetItBe: freeze records during initialization with `freeze: true`. ([@palkan][]) + +- Add FactoryDefault profiler (factory associations profilers). ([@palkan][]) + +- FactoryDefault: Allow creating a default per trait (or set of traits). ([@palkan][]) + +Now `create_default(:user)` and `create_default(:user, :admin)` would result into two defaults corresponding to the specified traits. + +- FactoryDefault: Add stats support. ([@palkan][]) + +Now you can see how often the default factory values have been used by specifying +the `FACTORY_DEFAULT_SUMMARY=1` or `FACTORY_DEFAULT_STATS=1` env var. + +- Support using FactoryDefault with before_all/let_it_be. ([@palkan][]) + +Currently, RSpec only. Default factories created within `before_all` or `let_it_be` are not reset 'till the end of the corresponding context. Thus, now it's possible to use `create_default` within `let_it_be` without any additional hacks. + +- FactoryDefault: Add `preserve_attributes = false | true` option. ([@palkan][]) + +Allow skipping defaults if association is defined with overrides, e.g.: + +```ruby +factory :post do + association :user, name: "Post Author" +end +``` + +- FactoryDefault: Add `skip_factory_default(&block)` to temporary disable default factories. ([@palkan][]) + +You can also use `TestProf::FactoryDefault.disable!(&block)`. + +- Add support for global `before_all` tags ([@maxshend][]) + +```ruby +TestProf::BeforeAll.configure do |config| + config.before(:begin, reset_sequences: true, foo: :bar) do + warn <<~MESSAGE + Do NOT create objects outside of transaction + because all db sequences will be reset to 1 + in every single example, so that IDs of new objects + can get into conflict with the long-living ones. + MESSAGE + end +end +``` + +## 1.0.11 (2022-10-27) + +- Fix monitoring methods with keyword args in Ruby 3+. ([@palkan][]) + +- Disable garbage collection frames when `TEST_STACK_PROF_IGNORE_GC` env variable is set ([@cbliard][]) + +- Fixed restoring lock_thread value in nested contexts ([@ygelfand][]) + +## 1.0.10 (2022-08-12) + +- Allow overriding global logger. ([@palkan][]) + +```ruby +require "test_prof/recipes/logging" + +TestProf::Rails::LoggingHelpers.logger = CustomLogger.new +``` + +## 1.0.9 (2022-05-05) + +- Add `AnyFixture.before_fixtures_reset` and `AnyFixture.after_fixtures_reset` callbacks. ([@ruslanshakirov][]) + +- Fixes ActiveRecord 6.1 issue with AnyFixture and Postgres config ([@markedmondson][]) + +## 1.0.8 (2022-03-11) + +- Restore the lock_thread value after rollback. ([@cou929][]) + +- Fixes the configuration of a printer for factory_prof runs + +- Ensure that defaults are stored in a threadsafe manner + +## 1.0.7 (2021-08-30) + +- Fix access to `let_it_be` variables in `after(:all)` hook. ([@cbarton][]) + +- Add support for using the before_all hook with Rails' parallelize feature (using processes). ([@peret][]) + +Make sure to include `TestProf::BeforeAll::Minitest` before you call `parallelize`. ## 1.0.6 (2021-06-23) @@ -52,7 +274,7 @@ And for every test run see the overall factories usage: ## 1.0.0.rc2 (2021-01-06) -- Make Rails fixtures accesible in `before_all`. ([@palkan][]) +- Make Rails fixtures accessible in `before_all`. ([@palkan][]) You can load and access fixtures when explicitly enabling them via `before_all(setup_fixtures: true, &block)`. @@ -113,7 +335,7 @@ end See more in [#181](https://github.com/test-prof/test-prof/issues/181). -- Adds the ability to define stackprof's interval sampling by using `TEST_STACK_PROF_INTERVAL` env variable ([@LynxEyes][]) +- Adds the ability to define stackprof interval sampling by using `TEST_STACK_PROF_INTERVAL` env variable ([@LynxEyes][]) Now you can use `$ TEST_STACK_PROF=1 TEST_STACK_PROF_INTERVAL=10000 rspec` to define a custom interval (in microseconds). @@ -191,7 +413,7 @@ end - Add threshold and custom event support to FactoryDoctor. ([@palkan][]) ```sh -$ FDOC=1 FDOC_EVENT="sql.rom" FDOC_THRESHOLD=0.1 rspec +FDOC=1 FDOC_EVENT="sql.rom" FDOC_THRESHOLD=0.1 rspec ``` - Add Fabrication support to FactoryDoctor. ([@palkan][]) @@ -240,16 +462,6 @@ end See [changelog](https://github.com/test-prof/test-prof/blob/v0.8.0/CHANGELOG.md) for versions <0.9.0. [@palkan]: https://github.com/palkan -[@marshall-lee]: https://github.com/marshall-lee -[@danielwestendorf]: https://github.com/danielwestendorf -[@shkrt]: https://github.com/Shkrt -[@idolgirev]: https://github.com/IDolgirev -[@desoleary]: https://github.com/desoleary -[@rabotyaga]: https://github.com/rabotyaga -[@vasfed]: https://github.com/Vasfed -[@szemek]: https://github.com/szemek -[@mkldon]: https://github.com/mkldon -[@dmagro]: https://github.com/dmagro [@danielwaterworth]: https://github.com/danielwaterworth [@envek]: https://github.com/Envek [@tyleriguchi]: https://github.com/tyleriguchi @@ -259,3 +471,21 @@ See [changelog](https://github.com/test-prof/test-prof/blob/v0.8.0/CHANGELOG.md) [@stefkin]: https://github.com/stefkin [@jaimerson]: https://github.com/jaimerson [@alexvko]: https://github.com/alexvko +[@cou929]: https://github.com/cou929 +[@ruslanshakirov]: https://github.com/ruslanshakirov +[@ygelfand]: https://github.com/ygelfand +[@cbliard]: https://github.com/cbliard +[@maxshend]: https://github.com/maxshend +[@rutgerw]: https://github.com/rutgerw +[@markedmondson]: https://github.com/markedmondson +[@cbarton]: https://github.com/cbarton +[@peret]: https://github.com/peret +[@bf4]: https://github.com/bf4 +[@Vankiru]: https://github.com/Vankiru +[@uzushino]: https://github.com/uzushino +[@lioneldebauge]: https://github.com/lioneldebauge +[@lHydra]: https://github.com/lHydra +[@john-h-k]: https://github.com/john-h-k +[@devinburnette]: https://github.com/devinburnette +[@elasticspoon]: https://github.com/elasticspoon +[@Rylan12]: https://github.com/Rylan12 diff --git a/Gemfile b/Gemfile index 6081092e..cf48827d 100644 --- a/Gemfile +++ b/Gemfile @@ -11,27 +11,29 @@ if File.exist?(local_gemfile) eval_gemfile(local_gemfile) # rubocop:disable Security/Eval else platform :mri do - gem "sqlite3", "~> 1.4" - gem "pg", "~> 1.0" + gem "sqlite3", "~> 2.0" end platform :jruby do - gem "activerecord-jdbcsqlite3-adapter", "~> 60.0" - gem "activerecord", "~> 6.0" + gem "activerecord-jdbcsqlite3-adapter" end - gem "activerecord", "~> 6.0" + gem "pg" + gem "activerecord", "~> 7.0" + gem "actionview" + gem "actionpack" gem "activerecord-import" - gem "factory_bot", "~> 5.0" + gem "factory_bot", ">= 6.0" gem "fabrication" gem "sidekiq", "~> 6.0" gem "timecop", "~> 0.9.1" platform :mri do - gem "pry-byebug" - gem "ruby-prof", ">= 0.16.0" + gem "debug" unless ENV["CI"] + gem "ruby-prof", ">= 1.4.0" gem "stackprof", ">= 0.2.9" + gem "vernier" end end diff --git a/README.md b/README.md index 223029d0..6774ef87 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,9 @@ And that's it) Supported Ruby versions: -- Ruby (MRI) >= 2.5.0 (**NOTE:** for Ruby 2.2 use TestProf < 0.7.0, Ruby 2.3 use TestProf ~> 0.7.0, Ruby 2.4 use TestProf <0.12.0) +- Ruby (MRI) >= 2.7.0 (**NOTE:** for Ruby 2.2 use TestProf < 0.7.0, Ruby 2.3 use TestProf ~> 0.7.0, Ruby 2.4 use TestProf <0.12.0, Ruby 2.5-2.6 use TestProf < 1.3) -- JRuby >= 9.1.0.0 (**NOTE:** refinements-dependent features might require 9.2.7+) +- JRuby >= 9.3.0 Supported RSpec version (for RSpec features only): >= 3.5.0 (for older RSpec versions use TestProf < 0.8.0). @@ -93,7 +93,7 @@ Supported RSpec version (for RSpec features only): >= 3.5.0 (for older RSpec ver Check out our [docs][]. -## What's next? +## What's next Have an idea? [Propose](https://github.com/test-prof/test-prof/issues/new) a feature request! diff --git a/docs/README.md b/docs/README.md index efedef32..ec5387a8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ > Ruby tests profiling and optimization toolbox + title="TestProf logo" class="home-logo" src="/assets/images/logo.svg"> TestProf is a collection of different tools to analyze your test suite performance. @@ -32,7 +32,7 @@ TestProf toolbox aims to help you identify bottlenecks in your test suite. It co

- TestProf map + TestProf map

@@ -47,16 +47,25 @@ TestProf toolbox aims to help you identify bottlenecks in your test suite. It co - [Discourse](https://github.com/discourse/discourse) reduced [~27% of their test suite time](https://twitter.com/samsaffron/status/1125602558024699904) - [Gitlab](https://gitlab.com/gitlab-org/gitlab-ce) reduced [39% of their API tests time](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14370) -- [CodeTriage](https://github.com/codetriage/codetriage) +- [Mastodon](https://github.com/mastodon/mastodon) - [Dev.to](https://github.com/thepracticaldev/dev.to) +- [CodeTriage](https://github.com/codetriage/codetriage) - [Open Project](https://github.com/opf/openproject) - [...and others](https://github.com/test-prof/test-prof/issues/73) ## Resources +- [From slow to go: Rails test profiling hands-on](https://evilmartians.com/events/from-slow-to-go-rails-test-profiling-hands-on-railsconf-2024) + +- [Profiling Ruby tests with Swiss precision](https://evilmartians.com/events/profiling-ruby-tests-with-swiss-precision-helvetic-ruby) + - [TestProf: a good doctor for slow Ruby tests](https://evilmartians.com/chronicles/testprof-a-good-doctor-for-slow-ruby-tests) -- [TestProf II: Factory therapy for your Ruby tests](https://evilmartians.com/chronicles/testprof-2-factory-therapy-for-your-ruby-tests-rspec-minitest) +- [TestProf II: factory therapy for your Ruby tests](https://evilmartians.com/chronicles/testprof-2-factory-therapy-for-your-ruby-tests-rspec-minitest) + +- [TestProf III: guided and automated Ruby test profiling](https://evilmartians.com/chronicles/test-prof-3-guided-and-automated-ruby-test-profiling) + +- [Rails Testing on Rocket Fuel: How we made our tests 5x faster](https://www.zerogravity.co.uk/blog/ruby-on-rails-slow-tests) - Paris.rb, 2018, "99 Problems of Slow Tests" talk [[video](https://www.youtube.com/watch?v=eDMZS_fkRtk), [slides](https://speakerdeck.com/palkan/paris-dot-rb-2018-99-problems-of-slow-tests)] diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 7d7bb64d..0fba455a 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -1,15 +1,20 @@ * [Getting Started](/getting_started.md) +* [Playbook](/playbook.md) + * Profilers - * [RubyProf Integration](/profilers/ruby_prof.md) - * [StackProf Integration](/profilers/stack_prof.md) + * [Ruby profilers](/profilers/ruby_profilers.md) * [Event Profiler](/profilers/event_prof.md) * [Tag Profiler](/profilers/tag_prof.md) * [Factory Doctor](/profilers/factory_doctor.md) * [Factory Profiler](/profilers/factory_prof.md) * [RSpecDissect Profiler](/profilers/rspec_dissect.md) + * [Memory Profiler](/profilers/memory_prof.md) * Recipes * [`before_all` Hook](/recipes/before_all.md) diff --git a/docs/assets/images/coggle.png b/docs/assets/images/coggle.png index b3ffe00b..422071f8 100644 Binary files a/docs/assets/images/coggle.png and b/docs/assets/images/coggle.png differ diff --git a/docs/misc/rubocop.md b/docs/misc/rubocop.md index 14800e82..9b42ef5f 100644 --- a/docs/misc/rubocop.md +++ b/docs/misc/rubocop.md @@ -1,15 +1,24 @@ # Custom RuboCop Cops -TestProf comes with the [RuboCop](https://github.com/bbatsov/rubocop) cops that help you write more performant tests. +TestProf comes with the [RuboCop](https://github.com/rubocop/rubocop) cops that help you write more performant tests. -To enable them, require `test_prof/rubocop` in your RuboCop configuration: +To enable them, add `test-prof` in your RuboCop configuration: ```yml # .rubocop.yml -require: - - 'test_prof/rubocop' +plugins: + - 'test-prof' ``` +Or you can just use it dynamically: + +```sh +bundle exec rubocop --plugin 'test-prof' --only RSpec/AggregateExamples +``` + +> [!NOTE] +> The plugin system is supported in RuboCop 1.72+. In earlier versions, use `require` instead of `plugins`. + To configure cops to your needs: ```yml @@ -17,12 +26,6 @@ RSpec/AggregateExamples: AddAggregateFailuresMetadata: false ``` -Or you can just require it dynamically: - -```sh -bundle exec rubocop -r 'test_prof/rubocop' --only RSpec/AggregateExamples -``` - ## RSpec/AggregateExamples This cop encourages you to use one of the greatest features of the recent RSpec – aggregating failures within an example. diff --git a/docs/playbook.md b/docs/playbook.md new file mode 100644 index 00000000..d3ae5c49 --- /dev/null +++ b/docs/playbook.md @@ -0,0 +1,209 @@ +# Playbook + +This document aims to help you get started with profiling test suites and answers the following questions: which profiles to run first? How do we interpret the results to choose the next steps? Etc. + +**NOTE**: This document assumes you're working with a Ruby on Rails application and RSpec testing framework. The ideas can easily be translated into other frameworks. + +> πŸ“Ό Check out also the ["From slow to go" RailsConf 2024 workshop recording](https://evilmartians.com/events/from-slow-to-go-rails-test-profiling-hands-on-railsconf-2024) to see this playbook in action. + +## Step 0. Configuration basics + +Low-hanging configuration fruits: + +- Disable logging\* in testsβ€”it's useless. If you really need it, use our [logging utils](./recipes/logging.md). + +```ruby +config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil)) +config.log_level = :fatal +``` + +- Disable coverage and built-in profiling by default. Use env var to enable it (e.g., `COVERAGE=true`) + +\* Modern SSD hard drives make the overhead of file-based logging almost negligible. Still, we recommend disabling logging to make sure tests are not affected in any environment (e.g., Docker on MacOS). + +## Step 1. General profiling + +It helps to identify not-so-low hanging fruits. We recommend using [StackProf](./profilers/ruby_profilers.md#stack_prof) or [Vernier](./profilers/ruby_profilers.md#vernier), so you must install them first (if not yet): + +```sh +bundle add stackprof +# or +bundle add vernier +``` + +Configure TestProf to generate JSON profiles by default: + +```ruby +TestProf::StackProf.configure do |config| + config.format = "json" +end +``` + +We recommend using [speedscope](https://www.speedscope.app) to analyze these profiles. + +### Step 1.1. Application boot profiling + +```sh +TEST_STACK_PROF=boot rspec ./spec/some_spec.rb +``` + +**NOTE:** running a single spec/test is enough for this profiling. + +What to look for? Some examples: + +- No [Bootsnap](https://github.com/Shopify/bootsnap) used or not configured to cache everything (e.g., YAML files) +- Slow Rails initializers that are not needed in tests. Vernier's Rails hooks feature is especially useful in analyzing Rails initializers. + +### Step 1.2. Sampling tests profiling + +The idea is to run a random subset of tests multiple times to reveal some application-wide problems. You must enable the [sampling feature](./recipes/tests_sampling.md) first: + +```rb +# For RSpec in your spec_helper.rb +require "test_prof/recipes/rspec/sample" + +# For Minitest in your test_helper.rb +require "test_prof/recipes/minitest/sample" +``` + +Then run **multiple times** and analyze the obtained flamegraphs: + +```sh +SAMPLE=100 bin/rails test +# or +SAMPLE=100 bin/rspec +``` + +Common findings: + +- Encryption calls (`*crypt*`-whatever): relax the settings in the test env +- Log calls: are you sure you disabled logs? +- Databases: maybe there are some low-hanging fruits (like using DatabaseCleaner truncation for every test instead of transactions) +- Network: should not be there for unit tests, inevitable for browser tests; use [Webmock](https://github.com/bblimke/webmock) to disable HTTP calls completely. + +## Step 2. Narrow down the scope + +This is an important step for large codebases. We must prioritize quick fixes that bring the most value (time reduction) over dealing with complex, slow tests individually (even if they're the slowest ones). For that, we first identify the **types of tests** contributing the most to the overall run time. + +We use [TagProf](./profilers/tag_prof.md) for that: + +```sh +TAG_PROF=type TAG_PROF_FORMAT=html TAG_PROF_EVENT=sql.active_record,factory.create bin/rspec +``` + +Looking at the generated diagram, you can identify the two most time-consuming test types (usually models and/or controllers among them). + +We assume that it's easier to find a common slowness cause for the whole group and fix it than dealing with individual tests. Given that assumption, we continue the process only within the selected group (let's say, models). + +## Step 3. Specialized profiling + +Within the selected group, we can first perform quick event-based profiling via [EventProf](./profilers/event_prof.md). (Maybe, with sampling enabled as well). + +### Step 3.1. Dependencies configuration + +At this point, we may identify some misconfigured or misused dependencies/gems. Common examples: + +- Inlined Sidekiq jobs: + +```sh +EVENT_PROF=sidekiq.inline bin/rspec spec/models +``` + +- Wisper broadcasts ([patch required](https://gist.github.com/palkan/aa7035cebaeca7ed76e433981f90c07b)): + +```sh +EVENT_PROF=wisper.publisher.broadcast bin/rspec spec/models +``` + +- PaperTrail logs creation: + +Enable custom profiling: + +```rb +TestProf::EventProf.monitor(PaperTrail::RecordTrail, "paper_trail.record", :record_create) +TestProf::EventProf.monitor(PaperTrail::RecordTrail, "paper_trail.record", :record_destroy) +TestProf::EventProf.monitor(PaperTrail::RecordTrail, "paper_trail.record", :record_update) +``` + +Run tests: + +```sh +EVENT_PROF=paper_trail.record bin/rspec spec/models +``` + +See [the Sidekiq example](https://evilmartians.com/chronicles/testprof-a-good-doctor-for-slow-ruby-tests#background-jobs) on how to quickly fix such problems using [RSpecStamp](./recipes/rspec_stamp.md). + +### Step 3.2. Data generation + +Identify the slowest tests based on the amount of time spent in the database or factories (if any): + +```sh +# Database interactions +EVENT_PROF=sql.active_record bin/rspec spec/models + +# Factories +EVENT_PROF=factory.create bin/rspec spec/models +``` + +Now, we can narrow our scope further to the top 10 files from the generated reports. If you use factories, use the `factory.create` report. + +**TIP:** In RSpec, you can mark the slowest examples with a custom tag automatically using the following command: + +```sh +EVENT_PROF=factory.create EVENT_PROF_STAMP=slow:factory bin/rspec spec/models +``` + +## Step 4. Factories usage + +Identify the most used factories among the `slow:factory` tests: + +```sh +FPROF=1 bin/rspec --tag slow:factory +``` + +If you see some factories used much more times than the total number of examples, you deal with _factory cascades_. + +Visualize the cascades: + +```sh +FPROF=flamegraph bin/rspec --tag slow:factory +``` + +The visualization should help to identify the factories to be fixed. You find possible solutions in [this post](https://evilmartians.com/chronicles/testprof-2-factory-therapy-for-your-ruby-tests-rspec-minitest). + +### Step 4.1. Factory defaults + +One option to fix cascades produced by model associations is to use [factory defaults](./recipes/factory_default.md). To estimate the potential impact and identify factories to apply this pattern to, run the following profiler: + +```sh +FACTORY_DEFAULT_PROF=1 bin/rspec --tag slow:factory +``` + +Try adding `create_default` and measure the impact: + +```sh +FACTORY_DEFAULT_SUMMARY=1 bin/rspec --tag slow:factory + +# More hits β€” better +FactoryDefault summary: hit=11 miss=3 +``` + +### Step 4.2. Factory fixtures + +Back to the `FPROF=1` report, see if you have some records created for every example (typically, `user`, `account`, `team`). Consider replacing them with fixtures using [AnyFixture](./recipes/any_fixture.md). + +## Step 5. Reusable setup + +It's common to have the same setup shared across multiple examples. You can measure the time spent in `let` / `before` compared to the actual example time using [RSpecDissect](./profilers/rspec_dissect.md): + +```sh +RD_PROF=1 bin/rspec +``` + +Take a look at the slowest groups and try to replace `let`/`let!` with [let_it_be](./recipes/let_it_be.md) and `before` with [before_all](./recipes/before_all.md). + +**IMPORTANT:** Knapsack Pro users must be aware that per-example balancing eliminates the positive effect of using `let_it_be` / `before_all`. You must switch to per-file balancing while at the same time keeping your files smallβ€”that's how you can maximize the effect of using Test Prof optimizations. + +## Conclusion + +After applying the steps above to a given group of tests, you should develop the patterns and techniques optimized for your codebase. Then, all you need is to extrapolate them to other groups. Good luck! diff --git a/docs/profilers/event_prof.md b/docs/profilers/event_prof.md index bacaee06..3e39b2eb 100644 --- a/docs/profilers/event_prof.md +++ b/docs/profilers/event_prof.md @@ -51,6 +51,8 @@ EVENT_PROF='sql.active_record,perform.active_job' rspec ... ### Minitest +In Minitest 6+, you must first activate TestProf plugin by adding `Minitest.load :test_prof` in your test helper. + Use `EVENT_PROF` environment variable set to event name: ```sh @@ -155,9 +157,9 @@ end ### `"factory.create"` -FactoryGirl provides its own instrumentation ('factory_girl.run_factory'); but there is a caveat – it fires an event every time a factory is used, even when we use factory for nested associations. Thus it's not possible to calculate the total time spent in factories due to the double calculation. +FactoryBot provides its own instrumentation ('factory_bot.run_factory'); but there is a caveat – it fires an event every time a factory is used, even when we use factory for nested associations. Thus it's not possible to calculate the total time spent in factories due to the double calculation. -EventProf comes with a little patch for FactoryGirl which provides instrumentation only for top-level `FactoryGirl.create` calls. It is loaded automatically if you use `"factory.create"` event: +EventProf comes with a little patch for FactoryBot which provides instrumentation only for top-level `FactoryBot.create` calls. It is loaded automatically if you use `"factory.create"` event: ```sh EVENT_PROF=factory.create bundle exec rspec diff --git a/docs/profilers/factory_doctor.md b/docs/profilers/factory_doctor.md index 5a23af99..40844ccb 100644 --- a/docs/profilers/factory_doctor.md +++ b/docs/profilers/factory_doctor.md @@ -3,7 +3,7 @@ One common bad pattern that slows our tests down is unnecessary database manipulation. Consider a _bad_ example: ```ruby -# with FactoryBot/FactoryGirl +# with FactoryBot it "validates name presence" do user = create(:user) user.name = "" @@ -21,7 +21,7 @@ end Here we create a new user record, run all callbacks and validations and save it to the database. We don't need all these! Here is a _good_ example: ```ruby -# with FactoryBot/FactoryGirl +# with FactoryBot it "validates name presence" do user = build_stubbed(:user) user.name = "" @@ -36,7 +36,7 @@ it "validates name presence" do end ``` -Read more about [`build_stubbed`](https://robots.thoughtbot.com/use-factory-girls-build-stubbed-for-a-faster-test). +Read more about [`build_stubbed`](https://thoughtbot.com/blog/use-factory-bots-build-stubbed-for-a-faster-test). FactoryDoctor is a tool that helps you identify such _bad_ tests, i.e. tests that perform unnecessary database queries. @@ -73,7 +73,7 @@ end FactoryDoctor supports: -- FactoryGirl/FactoryBot +- FactoryBot - Fabrication. ### RSpec @@ -96,6 +96,8 @@ After running the command above all _potentially_ bad examples would be marked w ### Minitest +In Minitest 6+, you must first activate TestProf plugin by adding `Minitest.load :test_prof` in your test helper. + To activate FactoryDoctor use `FDOC` environment variable: ```sh diff --git a/docs/profilers/factory_prof.md b/docs/profilers/factory_prof.md index 510a2702..0689b510 100644 --- a/docs/profilers/factory_prof.md +++ b/docs/profilers/factory_prof.md @@ -12,21 +12,22 @@ Total top-level: 10286 Total time: 04:31.222 (out of 07.16.124) Total uniq factories: 119 - total top-level total time time per call top-level time name - 6091 2715 115.7671s 0.0426s 50.2517s user - 2142 2098 93.3152s 0.0444s 92.1915s post - ... + name total top-level total time time per call top-level time + + user 6091 2715 115.7671s 0.0426s 50.2517s + post 2142 2098 93.3152s 0.0444s 92.1915s + ... ``` It shows both the total number of the factory runs and the number of _top-level_ runs, i.e. not during another factory invocation (e.g. when using associations.) It also shows the time spent generating records with factories and the amount of time taken per factory call. -**NOTE**: FactoryProf only tracks the database-persisted factories. In case of FactoryGirl/FactoryBot these are the factories provided by using `create` strategy. In case of Fabrication - objects that created using `create` method. +**NOTE**: FactoryProf only tracks the database-persisted factories. In case of FactoryBot these are the factories provided by using `create` strategy. In case of Fabrication - objects that created using `create` method. ## Instructions -FactoryProf can be used with FactoryGirl/FactoryBot or Fabrication - application can be bundled with both gems at the same time. +FactoryProf can be used with FactoryBot or Fabrication - application can be bundled with both gems at the same time. To activate FactoryProf use `FPROF` environment variable: @@ -54,6 +55,129 @@ And for every test run see the overall factories usage: [TEST PROF INFO] Time spent in factories: 04:31.222 (54% of total time) ``` +### Variations + +You can also add _variations_ (such as traits, overrides) information to reports by providing the `FPROF_VARS=1` environment variable or enabling it in your code: + +```ruby +TestProf::FactoryProf.configure do |config| + config.include_variations = true +end +``` + +For example: + +```sh +$ FPROF=1 FPROF_VARS=1 bin/rails test + +... + +[TEST PROF INFO] Factories usage + +Total: 15285 +Total top-level: 10286 +Total time: 04:31.222 (out of 07.16.124) +Total uniq factories: 119 + + name total top-level total time time per call top-level time + + user 6091 2715 115.7671s 0.0426s 50.251s + - 5243 1989 84.231s 0.0412s 34.321s + .admin 823 715 15.767s 0.0466s 5.257s + [name,role] 25 11 7.671s 0.0666s 1.257s + post 2142 2098 93.315s 0.0444s 92.191s + _ 2130 2086 87.685s 0.0412s 88.191s + .draft[tags] 12 12 9.315s 0.164s 42.115s + ... +``` + +In the example above, `-` indicates a factory without traits or overrides (e.g., `create(:user)`), `.xxx` indicates a trait and `[a,b]` indicates the overrides keys, e.g., `create(:user, :admin)` is an `.admin` variation, while `create(:post, :draft, tags: ["a"])`β€”`.draft[tags]` + +#### Variations limit config + +When running FactoryProf, the output may contain a variant that is too long, which will distort the output. + +To avoid this and focus on the most important statistics you can specify a variations limit value. Then a special ID (`[...]`) will be shown instead of the variant with the number of traits/overrides exceeding the limit. + +To use variations limit parameter set `FPROF_VARIATIONS_LIMIT` environment variable to `N` (where `N` is a limit number): + +```sh +FPROF=1 FPROF_VARIATIONS_LIMIT=5 rspec + +# or +FPROF=1 FPROF_VARIATIONS_LIMIT=5 bundle exec rake test +``` + +Or you can set the limit parameter through the `FactoryProf` configuration: + +```ruby +TestProf::FactoryProf.configure do |config| + config.variations_limit = 5 +end +``` + +### Exporting profile results to a JSON file + +FactoryProf can save profile results as a JSON file. + +To use this feature, set the `FPROF` environment variable to `json`: + +```sh +FPROF=json rspec + +# or +FPROF=json bundle exec rake test +``` + +Example output: + +```sh +[TEST PROF INFO] Profile results to JSON: tmp/test_prof/test-prof.result.json +``` + +### Reducing the output + +When running FactoryProf, the output may contain a lot of lines for factories that has been used a few times. +To avoid this and focus on the most important statistics you can specify a threshold value. Then you will be shown the factories whose total number exceeds the threshold. + +To use threshold option set `FPROF_THRESHOLD` environment variable to `N` (where `N` is a threshold number): + +```sh +FPROF=1 FPROF_THRESHOLD=30 rspec + +# or +FPROF=1 FPROF_THRESHOLD=30 bundle exec rake test +``` + +Or you can set the threshold parameter through the `FactoryProf` configuration: + +```ruby +TestProf::FactoryProf.configure do |config| + config.threshold = 30 +end +``` + +### Truncate long factory-names + +When running FactoryProf on a codebase with long factory names, the table layout may break. To avoid this you can allow FactoryProf to truncate these names. + +To use truncation set `FPROF_TRUNCATE_NAMES` environment variable to `1`: + +```sh +FPROF=1 FPROF_TRUNCATE_NAMES=1 rspec + +# or +FPROF=1 FPROF_TRUNCATE_NAMES=1 bundle exec rake test +``` + +Or you can set the truncate_names parameter through the `FactoryProf` configuration: + +```ruby +TestProf::FactoryProf.configure do |config| + config.truncate_names = true +end +``` + ## Factory Flamegraph The most useful feature of FactoryProf is the _FactoryFlame_ report. That's the special interpretation of Brendan Gregg's [flame graphs](http://www.brendangregg.com/flamegraphs.html) which allows you to identify _factory cascades_. @@ -69,7 +193,7 @@ FPROF=flamegraph bundle exec rake test That's how a report looks like: -![](../assets/factory-flame.gif) +TagProf UI How to read this? diff --git a/docs/profilers/memory_prof.md b/docs/profilers/memory_prof.md new file mode 100644 index 00000000..5c9db7e5 --- /dev/null +++ b/docs/profilers/memory_prof.md @@ -0,0 +1,87 @@ +# MemoryProf + +MemoryProf tracks memory usage during your test suite run, and can help to detect test examples and groups that cause memory spikes. Memory profiling supports two metrics: RSS and allocations. + +Example output: + +```sh +[TEST PROF INFO] MemoryProf results + +Final RSS: 673KB + +Top 5 groups (by RSS): + +AnswersController (./spec/controllers/answers_controller_spec.rb:3) – +80KB (13.50%) +QuestionsController (./spec/controllers/questions_controller_spec.rb:3) – +32KB (9.08%) +CommentsController (./spec/controllers/comments_controller_spec.rb:3) – +16KB (3.27%) + +Top 5 examples (by RSS): + +destroys question (./spec/controllers/questions_controller_spec.rb:38) – +144KB (24.38%) +change comments count (./spec/controllers/comments_controller_spec.rb:7) – +120KB (20.00%) +change Votes count (./spec/shared_examples/controllers/voted_examples.rb:23) – +90KB (16.36%) +change Votes count (./spec/shared_examples/controllers/voted_examples.rb:23) – +64KB (12.86%) +fails (./spec/shared_examples/controllers/invalid_examples.rb:3) – +32KB (5.00%) +``` + +The examples block shows the amount of memory used by each example, and the groups block displays the memory allocated by other code defined in the groups. For example, RSpec groups may include heavy `before(:all)` (or `before_all`) setup blocks, so it is helpful to see which groups use the most amount of memory outside of their examples. + +## Instructions + +To activate MemoryProf with: + +### RSpec + +Use `TEST_MEM_PROF` environment variable to set which metric to use: + +```sh +TEST_MEM_PROF='rss' rspec ... +TEST_MEM_PROF='alloc' rake rspec ... +``` + +### Minitest + +In Minitest 6+, you must first activate TestProf plugin by adding `Minitest.load :test_prof` in your test helper. + +Use `TEST_MEM_PROF` environment variable to set which metric to use: + +```sh +TEST_MEM_PROF='rss' rake test +TEST_MEM_PROF='alloc' rspec ... +``` + +or use CLI options as well: + +```sh +# Run a specific file using CLI option +ruby test/my_super_test.rb --mem-prof=rss + +# Show the list of possible options: +ruby test/my_super_test.rb --help +``` + +## Configuration + +By default, MemoryProf tracks the top 5 examples and groups that use the largest amount of memory. +You can set how many examples/groups to display with the option: + +```sh +TEST_MEM_PROF='rss' TEST_MEM_PROF_COUNT=10 rspec ... +``` + +or with CLI options for Minitest: + +```sh +# Run a specific file using CLI option +ruby test/my_super_test.rb --mem-prof=rs --mem-prof-top-count=10 +``` + +## Supported Ruby Engines & OS + +Currently the allocation mode is not supported for JRuby. + +Since RSS depends on the OS, MemoryProf uses different tools to retrieve it: + +* Linux – `/proc/$pid/statm` file, +* macOS, Solaris, BSD – `ps`, +* Windows – `Get-Process`, requires PowerShell to be installed. diff --git a/docs/profilers/ruby_prof.md b/docs/profilers/ruby_prof.md index 56fb736e..013f6b7e 100644 --- a/docs/profilers/ruby_prof.md +++ b/docs/profilers/ruby_prof.md @@ -1,84 +1,2 @@ -# Profiling with RubyProf - -Easily integrate the power of [ruby-prof](https://github.com/ruby-prof/ruby-prof) into your test suite. - -## Instructions - -Install `ruby-prof` gem (>= 0.17): - -```ruby -# Gemfile -group :development, :test do - gem "ruby-prof", ">= 0.17.0", require: false -end -``` - -RubyProf profiler has two modes: `global` and `per-example`. - -You can activate the global profiling using the environment variable `TEST_RUBY_PROF`: - -```sh -TEST_RUBY_PROF=1 bundle exec rake test - -# or for RSpec -TEST_RUBY_PROF=1 rspec ... -``` - -Or in your code: - -```ruby -TestProf::RubyProf.run -``` - -TestProf provides a built-in shared context for RSpec to profile examples individually: - -```ruby -it "is doing heavy stuff", :rprof do - # ... -end -``` - -**NOTE:** per-example profiling doesn't work when the global profiling is activated. - -## Configuration - -The most useful configuration option is `printer` – it allows you to specify a RubyProf [printer](https://github.com/ruby-prof/ruby-prof#printers). - -You can specify a printer through environment variable `TEST_RUBY_PROF`: - -```sh -TEST_RUBY_PROF=call_stack bundle exec rake test -``` - -Or in your code: - -```ruby -TestProf::RubyProf.configure do |config| - config.printer = :call_stack -end -``` - -By default, we use `FlatPrinter`. - -**NOTE:** to specify the printer for per-example profiles use `TEST_RUBY_PROF_PRINTER` env variable ('cause using `TEST_RUBY_PROF` activates the global profiling). - -Also, you can specify RubyProf mode (`wall`, `cpu`, etc) through `TEST_RUBY_PROF_MODE` env variable. - -See [ruby_prof.rb](https://github.com/test-prof/test-prof/tree/master/lib/test_prof/ruby_prof.rb) for all available configuration options and their usage. - -### Methods Exclusion - -It's useful to exclude some methods from the profile to focus only on the application code. - -TestProf uses RubyProf [`exclude_common_methods!`](https://github.com/ruby-prof/ruby-prof/blob/e087b7d7ca11eecf1717d95a5c5fea1e36ea3136/lib/ruby-prof/profile/exclude_common_methods.rb) by default (disable it with `config.exclude_common_methods = false`). - -We exclude some other common methods and RSpec specific internal methods by default. -To disable TestProf-defined exclusions set `config.test_prof_exclusions_enabled = false`. - -You can specify custom exclusions through `config.custom_exclusions`, e.g.: - -```ruby -TestProf::RubyProf.configure do |config| - config.custom_exclusions = {User => %i[save save!]} -end -``` + +

Please follow this link.

diff --git a/docs/profilers/ruby_profilers.md b/docs/profilers/ruby_profilers.md new file mode 100644 index 00000000..60e9aaad --- /dev/null +++ b/docs/profilers/ruby_profilers.md @@ -0,0 +1,268 @@ +# Using with Ruby profilers + +Test Prof allows you to use general Ruby profilers to profile test suites without needing to write any profiling code yourself. +Just install the profiler library and run your tests! + +Supported profilers: + +- [StackProf](#stackprof) +- [Vernier](#vernier) +- [RubyProf](#rubyprof) + +## StackProf + +[StackProf][] is a sampling call-stack profiler for Ruby. + +Make sure you have `stackprof` in your dependencies: + +```ruby +# Gemfile +group :development, :test do + gem "stackprof", ">= 0.2.9", require: false +end +``` + +### Profiling the whole test suite with StackProf + +**NOTE:** It's recommended to use [test sampling](../recipes/tests_sampling.md) to generate smaller profiling reports. + +You can activate StackProf profiling by setting the `TEST_STACK_PROF` env variable: + +```sh +TEST_STACK_PROF=1 bundle exec rake test + +# or for RSpec +TEST_STACK_PROF=1 bundle exec rspec ... +``` + +At the end of the test run, you will see the message from Test Prof including paths to generated reports (raw StackProf format and JSON): + +```sh +... + +[TEST PROF INFO] StackProf report generated: tmp/test_prof/stack-prof-report-wall-raw-total.dump +[TEST PROF INFO] StackProf JSON report generated: tmp/test_prof/stack-prof-report-wall-raw-total.json +``` + +We recommend uploading JSON reports to [Speedscope][] and analyze flamegraphs. Otherwise, feel free to use the `stackprof` CLI +to manipulate the raw report. + +### Profiling individual examples with StackProf + +Test Prof provides a built-in shared context for RSpec to profile examples individually: + +```ruby +it "is doing heavy stuff", :sprof do + # ... +end +``` + +**NOTE:** per-example profiling doesn't work when the global (per-suite) profiling is activated. + +### Profiling application boot with StackProf + +The application boot time could also makes testing slower. Try to profile your boot process with StackProf using the following command: + +```sh +# pick some random spec (1 is enough) +$ TEST_STACK_PROF=boot bundle exec rspec ./spec/some_spec.rb + +... +[TEST PROF INFO] StackProf report generated: tmp/test_prof/stack-prof-report-wall-raw-boot.dump +[TEST PROF INFO] StackProf JSON report generated: tmp/test_prof/stack-prof-report-wall-raw-boot.json +``` + +### StackProf configuration + +You can change StackProf mode (which is `wall` by default) through `TEST_STACK_PROF_MODE` env variable. + +You can also change StackProf interval through `TEST_STACK_PROF_INTERVAL` env variable. +For modes `wall` and `cpu`, `TEST_STACK_PROF_INTERVAL` represents microseconds and will default to 1000 as per `stackprof`. +For mode `object`, `TEST_STACK_PROF_INTERVAL` represents allocations and will default to 1 as per `stackprof`. + +You can disable garbage collection frames by setting `TEST_STACK_PROF_IGNORE_GC` env variable. +Garbage collection time will still be present in the profile but not explicitly marked with +its own frame. + +See [stack_prof.rb](https://github.com/test-prof/test-prof/tree/master/lib/test_prof/stack_prof.rb) for all available configuration options and their usage. + +## Vernier + +[Vernier][] is next generation sampling profiler for Ruby. Give it a try and see if it can help in identifying test peformance bottlenecks! + +Make sure you have `vernier` in your dependencies: + +```ruby +# Gemfile +group :development, :test do + gem "vernier", ">= 0.3.0", require: false +end +``` + +### Profiling the whole test suite with Vernier + +**NOTE:** It's recommended to use [test sampling](../recipes/tests_sampling.md) to generate smaller profiling reports. + +You can activate Verner profiling by setting the `TEST_VERNIER` env variable: + +```sh +TEST_VERNIER=1 bundle exec rake test + +# or for RSpec +TEST_VERNIER=1 bundle exec rspec ... +``` + +At the end of the test run, you will see the message from Test Prof including the path to the generated report: + +```sh +... + +[TEST PROF INFO] Vernier report generated: tmp/test_prof/vernier-report-wall-raw-total.json +``` + +Use the [profile-viewer](https://github.com/tenderlove/profiler/tree/ruby) gem or upload your profiles to [vernier.prof](https://vernier.prof). Alternatively, you can use [profiler.firefox.com](https://profiler.firefox.com) which profile-viewer is a fork of. + +### Profiling individual examples with Vernier + +Test Prof provides a built-in shared context for RSpec to profile examples individually: + +```ruby +it "is doing heavy stuff", :vernier do + # ... +end +``` + +**NOTE:** per-example profiling doesn't work when the global (per-suite) profiling is activated. + +### Profiling application boot with Vernier + +You can also profile your application boot process: + +```sh +# pick some random spec (1 is enough) +TEST_VERNIER=boot bundle exec rspec ./spec/some_spec.rb +``` + +### Add markers from Active Support Notifications + +You can add more insights to the resulting report by adding event markers from Active Support Notifications: + +```sh +TEST_VERNIER=1 TEST_VERNIER_HOOKS=rails bundle exec rake test + +# or for RSpec +TEST_VERNIER=1 TEST_VERNIER_HOOKS=rails bundle exec rspec ... +``` + +Or you can set the hooks parameter through the `Vernier` configuration: + +```ruby +TestProf::Vernier.configure do |config| + config.hooks = :rails +end +``` + +## RubyProf + +Easily integrate the power of [ruby-prof](https://github.com/ruby-prof/ruby-prof) into your test suite. + +Make sure `ruby-prof` is installed: + +```ruby +# Gemfile +group :development, :test do + gem "ruby-prof", ">= 1.4.0", require: false +end +``` + +### Profiling the whole test suite with RubyProf + +**NOTE:** It's highly recommended to use [test sampling](../recipes/tests_sampling.md) to generate smaller profiling reports and avoid slow test runs (RubyProf has a signifact overhead). + +You can activate the global profiling using the environment variable `TEST_RUBY_PROF`: + +```sh +TEST_RUBY_PROF=1 bundle exec rake test + +# or for RSpec +TEST_RUBY_PROF=1 bundle exec rspec ... +``` + +At the end of the test run, you will see the message from Test Prof including paths to generated reports: + +```sh +[TEST PROF INFO] RubyProf report generated: tmp/test_prof/ruby-prof-report-flat-wall-total.txt +``` + +#### Skipping test suite boot + +**NOTE:** RSpec only. + +It could be usefule to exclude the application boot and tests load from the RubyProf report to analyze only tests being executed (so you don't have `Kernel#require` being one of the top slowest methods). + +For that, specify the `TEST_RUBY_PROF_BOOT=false` (or "0", or "f") env variable: + +```sh +$ TEST_RUBY_PROF=1 TEST_RUBY_PROF_BOOT=0 bundle exec rspec ... + +[TEST PROF] RubyProf enabled for examples + +... +``` + +### Profiling individual examples with RubyProf + +TestProf provides a built-in shared context for RSpec to profile examples individually: + +```ruby +it "is doing heavy stuff", :rprof do + # ... +end +``` + +**NOTE:** per-example profiling doesn't work when the global profiling is activated. + +### RubyProf configuration + +The most useful configuration option is `printer` – it allows you to specify a RubyProf [printer](https://github.com/ruby-prof/ruby-prof#printers). + +You can specify a printer through environment variable `TEST_RUBY_PROF`: + +```sh +TEST_RUBY_PROF=call_stack bundle exec rake test +``` + +Or in your code: + +```ruby +TestProf::RubyProf.configure do |config| + config.printer = :call_stack +end +``` + +By default, we use `FlatPrinter`. + +**NOTE:** to specify the printer for per-example profiles use `TEST_RUBY_PROF_PRINTER` env variable ('cause using `TEST_RUBY_PROF` activates the global profiling). + +Also, you can specify RubyProf mode (`wall`, `cpu`, etc) through `TEST_RUBY_PROF_MODE` env variable. + +See [ruby_prof.rb](https://github.com/test-prof/test-prof/tree/master/lib/test_prof/ruby_prof.rb) for all available configuration options and their usage. + +It's useful to exclude some methods from the profile to focus only on the application code. + +TestProf uses RubyProf [`exclude_common_methods!`](https://github.com/ruby-prof/ruby-prof/blob/e087b7d7ca11eecf1717d95a5c5fea1e36ea3136/lib/ruby-prof/profile/exclude_common_methods.rb) by default (disable it with `config.exclude_common_methods = false`). + +We exclude some other common methods and RSpec specific internal methods by default. +To disable TestProf-defined exclusions set `config.test_prof_exclusions_enabled = false`. + +You can specify custom exclusions through `config.custom_exclusions`, e.g.: + +```ruby +TestProf::RubyProf.configure do |config| + config.custom_exclusions = {User => %i[save save!]} +end +``` + +[StackProf]: https://github.com/tmm1/stackprof +[Speedscope]: https://www.speedscope.app +[Vernier]: https://github.com/jhawthorn/vernier diff --git a/docs/profilers/stack_prof.md b/docs/profilers/stack_prof.md index 06b65d02..013f6b7e 100644 --- a/docs/profilers/stack_prof.md +++ b/docs/profilers/stack_prof.md @@ -1,79 +1,2 @@ -# Profiling with StackProf - -[StackProf](https://github.com/tmm1/stackprof) is a sampling call-stack profiler for ruby. - -## Instructions - -Install `stackprof` gem (>= 0.2.9): - -```ruby -# Gemfile -group :development, :test do - gem "stackprof", ">= 0.2.9", require: false -end -``` - -StackProf profiler has 2 modes: `global` and `per-example`. - -You can activate global profiling using the environment variable `TEST_STACK_PROF`: - -```sh -TEST_STACK_PROF=1 bundle exec rake test - -# or for RSpec -TEST_STACK_PROF=1 rspec ... -``` - -Or in your code: - -```ruby -TestProf::StackProf.run -``` - -TestProf provides a built-in shared context for RSpec to profile examples individually: - -```ruby -it "is doing heavy stuff", :sprof do - # ... -end -``` - -**NOTE:** per-example profiling doesn't work when the global profiling is activated. - -## Report formats - -Stackprof provides a CLI tool to manipulate generated reports (e.g. convert to different formats). - -By default, Test Prof shows you a command\* to generate an HTML report for analyzing flamegraphs, so you should run it yourself. - -\* only if you're collecting _raw_ samples data, which is the default Test Prof behaviour. - -Sometimes it's useful to have a JSON report (e.g. to use it with [speedscope](https://www.speedscope.app)), but `stackprof` only supports this only since version [0.2.13](https://github.com/tmm1/stackprof/blob/master/CHANGELOG.md#0213). - -If you're using an older version of Stackprof, Test Prof can help in generating JSON reports from _raw_ dumps. For that, use `TEST_STACK_PROF_FORMAT=json` or configure the default format in your code: - -```ruby -TestProf::StackProf.configure do |config| - config.format = "json" -end -``` - -## Profiling application boot - -The application boot time could also makes testing slower. Try to profile your boot process with StackProf using the following command: - -```sh -TEST_STACK_PROF=boot rspec ./spec/some_spec.rb -``` - -**NOTE:** we recommend to analyze the boot time using flame graphs, that's why raw data collection is always on in `boot` mode. - -## Configuration - -You can change StackProf mode (which is `wall` by default) through `TEST_STACK_PROF_MODE` env variable. - -You can also change StackProf interval through `TEST_STACK_PROF_INTERVAL` env variable. -For modes `wall` and `cpu`, `TEST_STACK_PROF_INTERVAL` represents microseconds and will default to 1000 as per `stackprof`. -For mode `object`, `TEST_STACK_PROF_INTERVAL` represents allocations and will default to 1 as per `stackprof`. - -See [stack_prof.rb](https://github.com/test-prof/test-prof/tree/master/lib/test_prof/stack_prof.rb) for all available configuration options and their usage. + +

Please follow this link.

diff --git a/docs/profilers/tag_prof.md b/docs/profilers/tag_prof.md index e246f530..6d863da3 100644 --- a/docs/profilers/tag_prof.md +++ b/docs/profilers/tag_prof.md @@ -2,7 +2,7 @@ TagProf is a simple profiler which collects examples statistics grouped by a provided tag value. -That's pretty useful in conjunction with `rspec-rails` built-in feature – `infer_spec_types_from_location!` – which automatically adds `type` to examples metadata. +That's pretty useful in conjunction with `rspec-rails` built-in feature – `infer_spec_type_from_file_location!` – which automatically adds `type` to examples metadata. Example output: @@ -26,19 +26,53 @@ TAG_PROF=type TAG_PROF_FORMAT=html bundle exec rspec That's how a report looks like: -![TagProf UI](../assets/tag-prof.gif) +TagProf UI ## Instructions -TagProf can only be used with RSpec. +TagProf can be used with both RSpec and Minitest (limited support, see below). To activate TagProf use `TAG_PROF` environment variable: +With Rspec: + ```sh # Group by type TAG_PROF=type rspec ``` +With Minitest\*: + +```sh +# using pure ruby +TAG_PROF=type ruby + +# using Rails built-in task +TAG_PROF=type bin/rails test +``` + +NB: if another value than "type" is used for TAG_PROF environment variable it will be ignored silently in both Minitest and RSpec. + +\* In Minitest 6+, you must first activate TestProf plugin by adding `Minitest.load :test_prof` in your test helper. + +### Usage specificity with Minitest + +Minitest does not support the usage of tags by default. TagProf therefore groups statistics by direct subdirectories of the root test directory. It assumes root test directory is named either `spec` or `test`. + +When no root test directory can be found the test statistics will not be grouped with other tests. They will be displayed per test with a significant warning message in the report. + +Example: + +```sh +[TEST PROF INFO] TagProf report for type + + type time sql.active_record total %total %time avg + +__unknown__ 00:04.808 00:01.402 42 33.87 54.70 00:00.114 + controller 00:02.855 00:00.921 42 33.87 32.48 00:00.067 + model 00:01.127 00:00.446 40 32.26 12.82 00:00.028 +``` + ## Profiling events You can combine TagProf with [EventProf](./event_prof.md) to track not only the total time spent but also the time spent for the specified activities (through events): @@ -59,7 +93,7 @@ Example output: model 00:01.127 00:00.446 40 32.26 12.82 00:00.028 ``` -Multiple events are also supported. +Multiple events are also supported (comma-separated). ## Pro-Tip: More Types diff --git a/docs/recipes/any_fixture.md b/docs/recipes/any_fixture.md index 0916db0c..6818b30a 100644 --- a/docs/recipes/any_fixture.md +++ b/docs/recipes/any_fixture.md @@ -18,7 +18,7 @@ RSpec.shared_context "account", account: true do @account = TestProf::AnyFixture.register(:account) do # Do anything here, AnyFixture keeps track of affected DB tables # For example, you can use factories here - FactoryGirl.create(:account) + FactoryBot.create(:account) # or with Fabrication Fabricate(:account) @@ -28,11 +28,18 @@ RSpec.shared_context "account", account: true do end end - # Use .register here to track the usage stats (see below) - let(:account) { TestProf::AnyFixture.register(:account) } + # Use .cached to retrieve the fixiture record + let(:account) { TestProf::AnyFixture.cached(:account) } +end + +# You can enhance the existing database cleaning. Posts will be deleted before fixtures reset +TestProf::AnyFixture.before_fixtures_reset do + Post.delete_all +end - # Or hard-reload object if there is chance of in-place modification - let(:account) { Account.find(TestProf::AnyFixture.register(:account).id) } +# Or after reset +TestProf::AnyFixture.after_fixtures_reset do + Post.delete_all end # Then in your tests @@ -77,35 +84,40 @@ at_exit { TestProf::AnyFixture.clean } ## DSL -We provide an optional _syntactic sugar_ (through Refinement) to make it easier to define fixtures: +We provide an optional _syntactic sugar_ (through Refinement) to make it easier to define fixtures and use callbacks: ```ruby require "test_prof/any_fixture/dsl" -# Enable DSL -using TestProf::AnyFixture::DSL +# Enable DSL in RSpec +RSpec.configure do |config| + config.include TestProf::AnyFixture::DSL +end + +# Minitest +class MyBaseTestCase < Minitest::Test + include TestProf::AnyFixture::DSL +end # and then you can use `fixture` method (which is just an alias for `TestProf::AnyFixture.register`) -before(:all) { fixture(:account) } +before(:all) { fixture(:account) { create(:account) } } # You can also use it to fetch the record (instead of storing it in instance variable) let(:account) { fixture(:account) } -``` -## `ActiveRecord#refind` +# You can just use `before_fixtures_reset` or `after_fixtures_reset` callbacks +before_fixtures_reset { Post.delete_all } +after_fixtures_reset { Post.delete_all } +``` -TestProf also provides an extension to _hard-reload_ ActiveRecord objects: +Note that the `#fixture` method also _refinds_ Active Record objects on read, i.e., the following two expressions works similarly: ```ruby -# instead of -let(:account) { Account.find(fixture(:account).id) } - -# load refinement -require "test_prof/ext/active_record_refind" +let(:account) { fixture(:account) } -using TestProf::Ext::ActiveRecordRefind +# similar to -let(:account) { fixture(:account).refind } +let(:account) { Account.find(TestProf::AnyFixture.cached(:account).id) } ``` ## Temporary disable fixtures @@ -125,9 +137,14 @@ context "global state", :with_clean_fixture do end ``` -How does it work? It wraps the example group into a transaction (using [`before_all`](./before_all.md)) and calls `TestProf::AnyFixture.clean` before running the examples. +How does it work? It wraps the example group into a transaction (using [`before_all`](./before_all.md)) and calls `TestProf::AnyFixture.clean` and `TestProf::AnyFixture.disable!` before running the examples and then call `TestProf::AnyFixture.enable!` at the context exit. The `disable!`/`enable!` method toggle the cache state. That makes it possible to re-use blocks passed during registration like there is no AnyFixture: + +```ruby +# Here we create a new account if AnyFixture is disabled +let(:account) { fixture(:account) { create(:account) } } +``` -Thus, this context is a little bit _heavy_. Try to avoid such situations and write specs independent of the global state. +Reseting fixtures (i.e., delete data from the affected tables) can be _heavy_. Try to avoid such situations and write specs independent of the global state. ## Usage report @@ -169,7 +186,7 @@ RSpec.shared_context "account", account: true do TestProf::AnyFixture.register_dump("account") do # Do anything here, AnyFixture keeps track of affected DB tables # For example, you can use factories here - account = FactoryGirl.create(:account, name: "test") + account = FactoryBot.create(:account, name: "test") # or with Fabrication account = Fabricate(:account, name: "test") diff --git a/docs/recipes/before_all.md b/docs/recipes/before_all.md index 2a25feb2..e91f6bb4 100644 --- a/docs/recipes/before_all.md +++ b/docs/recipes/before_all.md @@ -59,6 +59,35 @@ Make sure to check the [Caveats section](#caveats) of this document for details. ## Instructions +### Multiple database support + +The ActiveRecord BeforeAll adapter will only start a transaction using ActiveRecord::Base connection. +If you want to ensure `before_all` can use multiple connections, you need to ensure the connection +classes are loaded before using `before_all`. + +For example, imagine you have `ApplicationRecord` and a separate database for user accounts: + +```ruby +class Users < AccountsRecord + # ... +end + +class Articles < ApplicationRecord + # ... +end +``` + +Then those two Connection Classes do need to be loaded before the tests are run: + +```ruby + +# Ensure connection classes are loaded +ApplicationRecord +AccountsRecord +``` + +This code can be added to `rails_helper.rb` or the rake tasks that runs minitests. + ### RSpec In your `rails_helper.rb` (or `spec_helper.rb` after *ActiveRecord* has been loaded): @@ -235,3 +264,20 @@ TestProf::BeforeAll.configure do |config| config.setup_fixtures = true end ``` + +## Global Tags + +You can register callbacks for specific RSpec Example Groups using tags: + +```ruby +TestProf::BeforeAll.configure do |config| + config.before(:begin, reset_sequences: true, foo: :bar) do + warn <<~MESSAGE + Do NOT create objects outside of transaction + because all db sequences will be reset to 1 + in every single example, so that IDs of new objects + can get into conflict with the long-living ones. + MESSAGE + end +end +``` diff --git a/docs/recipes/factory_all_stub.md b/docs/recipes/factory_all_stub.md index 5a84ab72..a7b88978 100644 --- a/docs/recipes/factory_all_stub.md +++ b/docs/recipes/factory_all_stub.md @@ -1,10 +1,10 @@ # FactoryAllStub -_Factory All Stub_ is a spell to force FactoryBot/FactoryGirl use only `build_stubbed` strategy (even if you call `create` or `build`). +_Factory All Stub_ is a spell to force FactoryBot use only `build_stubbed` strategy (even if you call `create` or `build`). The idea behind it is to quickly fix [Factory Doctor](../profilers/factory_doctor.md) offenses (and even do that automatically). -**NOTE**. Only works with FactoryGirl/FactoryBot. Should be considered only as a temporary specs fix. +**NOTE**. Only works with FactoryBot. Should be considered only as a temporary specs fix. ## Instructions @@ -25,7 +25,7 @@ TestProf::FactoryAllStub.enable! To disable _all-stub_ mode and use factories as always: ```ruby -TestProf::FactoryAllStub.enable! +TestProf::FactoryAllStub.disable! ``` ## RSpec diff --git a/docs/recipes/factory_default.md b/docs/recipes/factory_default.md index f2499ac6..832db42e 100644 --- a/docs/recipes/factory_default.md +++ b/docs/recipes/factory_default.md @@ -2,8 +2,6 @@ _FactoryDefault_ aims to help you cope with _factory cascades_ (see [FactoryProf](../profilers/factory_prof.md)) by reusing associated records. -**NOTE**. Only works with FactoryGirl/FactoryBot. - It can be very useful when you're working on a typical SaaS application (or other hierarchical data). Consider an example. Assume we have the following factories: @@ -28,6 +26,19 @@ factory :task do end ``` +Or in case of Fabrication: + +```ruby +Fabricator(:account) do +end + +Fabricator(:user) do + account +end + +# etc. +``` + And we want to test the `Task` model: ```ruby @@ -94,9 +105,9 @@ In your `spec_helper.rb`: require "test_prof/recipes/rspec/factory_default" ``` -This adds two new methods to FactoryBot: +This adds the following methods to FactoryBot and/or Fabrication: -- `FactoryBot#set_factory_default(factory, object)` – use the `object` as default for associations built with `factory` +- `FactoryBot#set_factory_default(factory, object)` / `Fabricate.set_fabricate_default(factory, object)` – use the `object` as default for associations built with `factory`. Example: @@ -104,17 +115,45 @@ Example: let(:user) { create(:user) } before { FactoryBot.set_factory_default(:user, user) } + +# You can also set the default factory with traits +FactoryBot.set_factory_default([:user, :admin], admin) + +# Or (since v1.4) +FactoryBot.set_factory_default(:user, :admin, admin) + +# You can also register a default record for specific attribute overrides +Fabricate.set_fabricate_default(:post, post, state: "draft") +``` + +- `FactoryBot#create_default(...)` / `Fabricate.create_default(...)` – is a shortcut for `create` + `set_factory_default`. + +- `FactoryBot#get_factory_default(factory)` / `Fabricate.get_fabricate_default(factory)` – retrieves the default value for `factory` (since v1.4). + +```rb +# This method also supports traits +admin = FactoryBot.get_factory_default(:user, :admin) ``` -- `FactoryBot#create_default(factory, *args)` – is a shortcut for `create` + `set_factory_default`. +**IMPORTANT:** Defaults are **cleaned up after each example** by default (i.e., when using `test_prof/recipes/rspec/factory_default`). + +### Using with `before_all` / `let_it_be` + +Defaults created within `before_all` and `let_it_be` are not reset after each example, but only at the end of the corresponding example group. So, it's possible to call `create_default` within `let_it_be` without any additional configuration. **RSpec only** + +**IMPORTANT:** You must load FactoryDefault after loading BeforeAll to make this feature work. -**NOTE**. Defaults are **cleaned up after each example** by default. That means you cannot create defaults within `before(:all)` / [`before_all`](./before_all.md) / [`let_it_be`](./let_it_be.md) definitions. That could be changed in the future, for now [check this workaround](https://github.com/test-prof/test-prof/issues/125#issuecomment-471706752). +**NOTE**. Regular `before(:all)` callbacks are not supported. ### Working with traits -When you have traits in your associations like: +You can use traits in your associations, for example: ```ruby +factory :comment do + user +end + factory :post do association :user, factory: %i[user able_to_post] end @@ -124,7 +163,119 @@ factory :view do end ``` -and set a default for `user` factory - you will find the same object used in all of the above factories. Sometimes this may break your logic. +If there is a default value for the `user` factory, it's gonna be used independently of traits. This may break your logic. + +To prevent this, configure FactoryDefault to preserve traits: + +```ruby +# Globally +TestProf::FactoryDefault.configure do |config| + config.preserve_traits = true +end + +# or in-place +create_default(:user, preserve_traits: true) +``` + +Creating a default with trait works as follows: + +```ruby +# Create a default with trait +user = create_default(:user_poster, :able_to_post) + +# When an association has no traits specified, the default with trait is used +create(:comment).user == user #=> true +# When an association has the matching trait specified, the default is used, too +create(:post).user == user #=> true +# When the association's trait differs, default is skipped +create(:view).user == user #=> false +``` + +### Handling attribute overrides + +It's possible to define attribute overrides for associations: + +```ruby +factory :post do + association :user, name: "Poster" +end + +factory :view do + association :user, name: "Viewer" +end +``` + +FactoryDefault ignores such overrides and still returns a default `user` record (if created). You can turn the attribute awareness feature on to skip the default record if overrides don't match the default object attributes: + +```ruby +# Globally +TestProf::FactoryDefault.configure do |config| + config.preserve_attributes = true +end + +# or in-place +create_default :user, preserve_attributes: true +``` + +**NOTE:** In the future versions of Test Prof, both `preserve_traits` and `preserve_attributes` will default to true. We recommend settings them to true if you just starting using this feature. + +### Ignoring default factories + +You can temporary disable the defaults usage by wrapping a code with the `skip_factory_default` method: + +```ruby +account = create_default(:account) +another_account = skip_factory_default { create(:account) } + +expect(another_account).not_to eq(account) +``` + +### Showing usage stats + +You can display the FactoryDefault usage stats by setting the `FACTORY_DEFAULT_SUMMARY=1` or `FACTORY_DEFAULT_STATS=1` env vars or by setting the configuration values: + +```ruby +TestProf::FactoryDefault.configure do |config| + config.report_summary = true + # Report stats prints the detailed usage information (including summary) + config.report_stats = true +end +``` + +For example: + +```sh +$ FACTORY_DEFAULT_SUMMARY=1 bundle exec rspec + +FactoryDefault summary: hit=11 miss=3 +``` + +Where `hit` indicates the number of times the default factory value was used instead of a new one when an association was created; `miss` indicates the number of time the default value was ignored due to traits or attributes mismatch. + +## Factory Default profiling, or when to use defaults + +Factory Default ships with the profiler, which can help you to see how associations are being used in your test suite, so you can decide on using `create_default` or not. + +To enable profiling, run your tests with the `FACTORY_DEFAULT_PROF=1` set: + +```sh +$ FACTORY_DEFAULT_PROF=1 bundle exec rspec spec/some/file_spec.rb + +..... + +[TEST PROF INFO] Factory associations usage: + + factory count total time + + user 17 00:42.010 + user[traited] 15 00:31.560 + user{tag:"some tag"} 1 00:00.205 + +Total associations created: 33 +Total uniq associations created: 3 +Total time spent: 01:13.775 +``` + +Since default factories are usually registered per an example group (or test class), we recommend running this profiler against a particular file, so you can quickly identify the possibility of adding `create_default` and improve the tests speed. -To prevent this - set `FactoryDefault.preserve_traits = true` or use per-factory override -`create_default(:user, preserve_traits: true)`. This reverts back to original FactoryBot behavior for associations that have explicit traits defined. +**NOTE:** You can also use the profiler to measure the effect of adding `create_default`; for that, compare the results of running the profiler with FactoryDefault enabled and disabled (you can do that by passing the `FACTORY_DEFAULT_DISABLED=1` env var). diff --git a/docs/recipes/let_it_be.md b/docs/recipes/let_it_be.md index 7eaad8c0..e09ae0d7 100644 --- a/docs/recipes/let_it_be.md +++ b/docs/recipes/let_it_be.md @@ -266,3 +266,36 @@ end And then tag contexts/examples with `:let_it_be_frost` to enable this feature. Alternatively, you can specify `freeze` modifier explicitly (`let_it_be(freeze: true)`) or configure an alias. + +## Report duplicates + +Although we suggest using `let_it_be` instead of `let!`, there is one important difference: you can override `let!` definition with the same or nested context, so only the latter one is called; `let_it_be` records could be overridden, but still created. For example: + +```ruby +context "A" do + let!(:user) { create(:user, name: "a") } + let_it_be(:post) { create(:post, title: "A") } + + specify { expect(User.all.pluck(:name)).to eq ["a"] } + specify { expect(Post.all.pluck(:title)).to eq ["A"] } + + context "B" do + let!(:user) { create(:user, name: "b") } + let_it_be(:post) { create(:post, title: "B") } + + specify { expect(User.all.pluck(:name)).to eq ["b"] } + specify { expect(Post.all.pluck(:title)).to eq ["B"] } # fails, because there are two posts + end +end +``` + +So for your convenience, you can configure the behavior when let_it_be is overridden. + +```ruby +TestProf::LetItBe.configure do |config| + config.report_duplicates = :warn # Rspec.warn_with + config.report_duplicates = :raise # Kernel.raise +end +``` + +By default this parameter is disabled. You can configure the behavior that will generate a warning or raise an exception. diff --git a/forspell.dict b/forspell.dict index b9d72081..5e80eec8 100644 --- a/forspell.dict +++ b/forspell.dict @@ -19,3 +19,16 @@ stackprof Stackprof Tigeot integrations +Bootsnap +Webmock +utils +misconfigured +Inlined +Wisper +Pennylane +Makar +Ermokhin +Burkhard +Sistovaris +Timecop +refind diff --git a/gemfiles/activerecord6.gemfile b/gemfiles/activerecord6.gemfile index b4897985..db615795 100644 --- a/gemfiles/activerecord6.gemfile +++ b/gemfiles/activerecord6.gemfile @@ -8,4 +8,6 @@ gem "sidekiq", "~> 6.0" gem "timecop", "~> 0.9.1" gem "pg" +gem "logger" + gemspec path: '..' diff --git a/gemfiles/activerecord60.gemfile b/gemfiles/activerecord60.gemfile deleted file mode 100644 index ff49b351..00000000 --- a/gemfiles/activerecord60.gemfile +++ /dev/null @@ -1,11 +0,0 @@ -source "https://rubygems.org" - -gem "activerecord", "~> 6.0.0" -gem "factory_bot", "~> 5.0" -gem "fabrication" -gem "sqlite3", "~> 1.4" -gem "sidekiq", "~> 6.0" -gem "timecop", "~> 0.9.1" -gem "pg" - -gemspec path: ".." diff --git a/gemfiles/activerecord7.gemfile b/gemfiles/activerecord7.gemfile new file mode 100644 index 00000000..6762f26f --- /dev/null +++ b/gemfiles/activerecord7.gemfile @@ -0,0 +1,11 @@ +source 'https://rubygems.org' + +gem "activerecord", "~> 7.0" +gem "factory_bot" +gem "fabrication" +gem "sqlite3", "~> 2.0" +gem "sidekiq" +gem "timecop" +gem "pg" + +gemspec path: '..' diff --git a/gemfiles/activerecord72.gemfile b/gemfiles/activerecord72.gemfile new file mode 100644 index 00000000..bc7e7243 --- /dev/null +++ b/gemfiles/activerecord72.gemfile @@ -0,0 +1,11 @@ +source 'https://rubygems.org' + +gem "rails", github: "rails/rails", branch: "7-2-stable" +gem "factory_bot" +gem "fabrication" +gem "sqlite3", "~> 2.0" +gem "sidekiq" +gem "timecop" +gem "pg" + +gemspec path: '..' diff --git a/gemfiles/activerecord8.gemfile b/gemfiles/activerecord8.gemfile new file mode 100644 index 00000000..585dd436 --- /dev/null +++ b/gemfiles/activerecord8.gemfile @@ -0,0 +1,12 @@ +source 'https://rubygems.org' + +gem "activerecord", "~> 8.0" +gem "factory_bot" +gem "fabrication" +gem "sqlite3", "~> 2.0" +gem "sidekiq" +gem "timecop" +gem "pg" +gem "benchmark" # ref: https://github.com/aderyabin/sniffer/pull/73 + +gemspec path: '..' diff --git a/gemfiles/jruby.gemfile b/gemfiles/jruby.gemfile index 0c1d00aa..86418fdf 100644 --- a/gemfiles/jruby.gemfile +++ b/gemfiles/jruby.gemfile @@ -1,16 +1,13 @@ source 'https://rubygems.org' -gem "activerecord-jdbcsqlite3-adapter", "~> 52.0" +gem "activerecord-jdbcsqlite3-adapter" gem "jdbc-sqlite3" -gem "activerecord", "~> 5.2" -gem "activerecord-import" -# See https://github.com/ruby-i18n/i18n/issues/555 -gem "i18n", "1.8.7" +gem "activerecord", "~> 7.1.0" -gem "factory_bot", "~> 5.0" +gem "factory_bot" gem "fabrication" -gem "sidekiq", "~> 5.0" -gem "timecop", "~> 0.9.1" +gem "sidekiq" +gem "timecop" gemspec path: '..' diff --git a/gemfiles/railsmaster.gemfile b/gemfiles/railsmaster.gemfile index 1fce300d..b736cb7f 100644 --- a/gemfiles/railsmaster.gemfile +++ b/gemfiles/railsmaster.gemfile @@ -1,12 +1,14 @@ source "https://rubygems.org" -gem "rails", github: "rails/rails" +gem "rails", github: "rails/rails", branch: "main" gem "fabrication" -gem "factory_bot", "~> 5.0" -gem "sqlite3", "~> 1.4.0" +gem "factory_bot", ">= 6" +gem "sqlite3", "~> 2.0" gem "sidekiq", "~> 4.0" gem "timecop", "~> 0.9.1" gem "pg" +gem "mysql2" +gem "benchmark" # ref: https://github.com/aderyabin/sniffer/pull/73 gemspec path: ".." diff --git a/gemfiles/activerecord5.gemfile b/gemfiles/rspecrails4.gemfile similarity index 72% rename from gemfiles/activerecord5.gemfile rename to gemfiles/rspecrails4.gemfile index 0803da34..267efca2 100644 --- a/gemfiles/activerecord5.gemfile +++ b/gemfiles/rspecrails4.gemfile @@ -1,12 +1,13 @@ source 'https://rubygems.org' -gem "activerecord", "~> 5.0" -gem "activerecord-import" +gem "activerecord", "~> 6.0" gem "factory_bot", "~> 5.0" gem "fabrication" gem "sqlite3", "~> 1.4" gem "sidekiq", "~> 6.0" gem "timecop", "~> 0.9.1" gem "pg" +gem "rspec-rails", "~> 4.0" +gem "logger" gemspec path: '..' diff --git a/gemfiles/rubocop.gemfile b/gemfiles/rubocop.gemfile index aa25bbd3..43c947a2 100644 --- a/gemfiles/rubocop.gemfile +++ b/gemfiles/rubocop.gemfile @@ -1,4 +1,5 @@ source "https://rubygems.org" do - gem "rubocop-md", "~> 1.0" + gem "rubocop-md", "~> 2.0" gem "standard", "~> 1.0" + gem "lint_roller", "~> 1.0" end diff --git a/lefthook.yml b/lefthook.yml deleted file mode 100644 index ca5678ca..00000000 --- a/lefthook.yml +++ /dev/null @@ -1,18 +0,0 @@ -pre-commit: - commands: - mdl: - tags: style - glob: "**/*.md" - run: mdl {staged_files} - liche: - tags: links - glob: "*.md" - run: liche -d docs/ -r docs/* CHANGELOG.md README.md && test "{staged_files}" - forspell: - tags: grammar - glob: "**/*.md" - run: forspell {staged_files} - rubocop: - tags: style - glob: "**/*.md" - run: BUNDLE_GEMFILE=gemfiles/rubocop.gemfile bundle exec rubocop {staged_files} diff --git a/lib/minitest/test_prof_plugin.rb b/lib/minitest/test_prof_plugin.rb index a21be2a7..30ea1d43 100644 --- a/lib/minitest/test_prof_plugin.rb +++ b/lib/minitest/test_prof_plugin.rb @@ -2,6 +2,8 @@ require "test_prof/event_prof/minitest" require "test_prof/factory_doctor/minitest" +require "test_prof/memory_prof/minitest" +require "test_prof/tag_prof/minitest" module Minitest # :nodoc: module TestProf # :nodoc: @@ -13,6 +15,9 @@ def self.configure_options(options = {}) opts[:per_example] = true if ENV["EVENT_PROF_EXAMPLES"] opts[:fdoc] = true if ENV["FDOC"] opts[:sample] = true if ENV["SAMPLE"] || ENV["SAMPLE_GROUPS"] + opts[:mem_prof_mode] = ENV["TEST_MEM_PROF"] if ENV["TEST_MEM_PROF"] + opts[:mem_prof_top_count] = ENV["TEST_MEM_PROF_COUNT"] if ENV["TEST_MEM_PROF_COUNT"] + opts[:tag_prof] = true if ENV["TAG_PROF"] == "type" end end end @@ -33,6 +38,12 @@ def self.plugin_test_prof_options(opts, options) opts.on "--factory-doctor", TrueClass, "Enable Factory Doctor for your examples" do |flag| options[:fdoc] = flag end + opts.on "--mem-prof=MODE", "Enable MemoryProf for your examples" do |flag| + options[:mem_prof_mode] = flag + end + opts.on "--mem-prof-top-count=N", "Limits MemoryProf results with N groups/examples" do |flag| + options[:mem_prof_top_count] = flag + end end def self.plugin_test_prof_init(options) @@ -40,6 +51,8 @@ def self.plugin_test_prof_init(options) reporter << TestProf::EventProfReporter.new(options[:io], options) if options[:event] reporter << TestProf::FactoryDoctorReporter.new(options[:io], options) if options[:fdoc] + reporter << TestProf::MemoryProfReporter.new(options[:io], options) if options[:mem_prof_mode] + reporter << Minitest::TestProf::TagProfReporter.new(options[:io], options) if options[:tag_prof] ::TestProf::MinitestSample.call if options[:sample] end diff --git a/lib/rubocop/test_prof.rb b/lib/rubocop/test_prof.rb new file mode 100644 index 00000000..b58990ae --- /dev/null +++ b/lib/rubocop/test_prof.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +if Gem::Version.new(RuboCop::Version::STRING) < Gem::Version.new("0.51.0") + warn "TestProf cops require RuboCop >= 0.51.0 to run." + return +end + +require "rubocop/test_prof/plugin" +require "rubocop/test_prof/cops/rspec/aggregate_examples" diff --git a/lib/test_prof/cops/rspec/aggregate_examples.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples.rb similarity index 90% rename from lib/test_prof/cops/rspec/aggregate_examples.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples.rb index 5c031091..3f58f58d 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples.rb @@ -108,7 +108,8 @@ module RSpec # expect(number).to be_odd # end # - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base + extend AutoCorrector include LineRangeHelpers include MetadataHelpers include NodeMatchers @@ -123,29 +124,22 @@ class AggregateExamples < ::RuboCop::Cop::Cop def on_block(node) example_group_with_several_examples(node) do |all_examples| example_clusters(all_examples).each do |_, examples| - examples[1..-1].each do |example| + examples.drop(1).each do |example| add_offense(example, - location: :expression, - message: message_for(example, examples[0])) + message: message_for(example, examples[0])) do |corrector| + clusters = example_clusters_for_autocorrect(example) + clusters.each do |metadata, examples| + range = range_for_replace(examples) + replacement = aggregated_example(examples, metadata) + corrector.replace(range, replacement) + examples.drop(1).map { |example| drop_example(corrector, example) } + end + end end end end end - def autocorrect(example_node) - clusters = example_clusters_for_autocorrect(example_node) - return if clusters.empty? - - lambda do |corrector| - clusters.each do |metadata, examples| - range = range_for_replace(examples) - replacement = aggregated_example(examples, metadata) - corrector.replace(range, replacement) - examples[1..-1].map { |example| drop_example(corrector, example) } - end - end - end - private # Clusters of examples in the same example group, on the same nesting diff --git a/lib/test_prof/cops/rspec/aggregate_examples/its.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/its.rb similarity index 87% rename from lib/test_prof/cops/rspec/aggregate_examples/its.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples/its.rb index a67033ae..979f618f 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples/its.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/its.rb @@ -3,7 +3,7 @@ module RuboCop module Cop module RSpec - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base # @example `its` # # # Supports regular `its` call with an attribute/method name, @@ -48,14 +48,15 @@ def new_body(node) def transform_its(body, arguments) argument = arguments.first - replacement = case argument.type - when :array - key = argument.values.first - "expect(subject[#{key.source}])" - else - property = argument.value - "expect(subject.#{property})" - end + replacement = + case argument.type + when :array + key = argument.values.first + "expect(subject[#{key.source}])" + else + property = argument.value + "expect(subject.#{property})" + end body.source.gsub(/is_expected|are_expected/, replacement) end @@ -63,7 +64,7 @@ def example_metadata(example) return super unless its?(example.send_node) # First parameter to `its` is not metadata. - example.send_node.arguments[1..-1] + example.send_node.arguments[1..] end def its?(node) diff --git a/lib/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb similarity index 93% rename from lib/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb index dc14a30e..ca37e10d 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/line_range_helpers.rb @@ -3,7 +3,7 @@ module RuboCop module Cop module RSpec - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base # @internal Support methods for keeping newlines around examples. module LineRangeHelpers include RangeHelp diff --git a/lib/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb similarity index 98% rename from lib/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb index e4b1dcb6..149b667d 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects.rb @@ -5,7 +5,7 @@ module RuboCop module Cop module RSpec - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base # When aggregated, the expectations will fail when not supposed to or # have a risk of not failing when expected to. One example is # `validate_presence_of :comment` as it leaves an empty comment after diff --git a/lib/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb similarity index 97% rename from lib/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb index 25b0c80d..291c0251 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/metadata_helpers.rb @@ -3,7 +3,7 @@ module RuboCop module Cop module RSpec - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base # @internal # Support methods for example metadata. # Examples with similar metadata are grouped. diff --git a/lib/test_prof/cops/rspec/aggregate_examples/node_matchers.rb b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/node_matchers.rb similarity index 97% rename from lib/test_prof/cops/rspec/aggregate_examples/node_matchers.rb rename to lib/rubocop/test_prof/cops/rspec/aggregate_examples/node_matchers.rb index 627ea82c..b44f2617 100644 --- a/lib/test_prof/cops/rspec/aggregate_examples/node_matchers.rb +++ b/lib/rubocop/test_prof/cops/rspec/aggregate_examples/node_matchers.rb @@ -5,7 +5,7 @@ module RuboCop module Cop module RSpec - class AggregateExamples < ::RuboCop::Cop::Cop + class AggregateExamples < ::RuboCop::Cop::Base # @internal # Node matchers and searchers. module NodeMatchers diff --git a/lib/test_prof/cops/rspec/language.rb b/lib/rubocop/test_prof/cops/rspec/language.rb similarity index 100% rename from lib/test_prof/cops/rspec/language.rb rename to lib/rubocop/test_prof/cops/rspec/language.rb diff --git a/lib/rubocop/test_prof/plugin.rb b/lib/rubocop/test_prof/plugin.rb new file mode 100644 index 00000000..e0d7d71f --- /dev/null +++ b/lib/rubocop/test_prof/plugin.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "lint_roller" + +module RuboCop + module TestProf + # A plugin that integrates TestProf with RuboCop's plugin system. + class Plugin < LintRoller::Plugin + def about + LintRoller::About.new( + name: "test-prof", + version: ::TestProf::VERSION, + homepage: "https://test-prof.evilmartians.io/misc/rubocop", + description: "RuboCop plugin to help you write more performant tests." + ) + end + + def supported?(context) + context.engine == :rubocop + end + + def rules(_context) + LintRoller::Rules.new( + type: :path, + config_format: :rubocop, + value: Pathname.new(__dir__).join("../../../config/default.yml") + ) + end + end + end +end diff --git a/lib/test-prof.rb b/lib/test-prof.rb index 52edfe11..cad15a9b 100644 --- a/lib/test-prof.rb +++ b/lib/test-prof.rb @@ -1,3 +1,8 @@ # frozen_string_literal: true require "test_prof" + +# For RuboCop plugin +module RuboCop + autoload :TestProf, "rubocop/test_prof" +end diff --git a/lib/test_prof.rb b/lib/test_prof.rb index eef6c49c..cff6fb40 100644 --- a/lib/test_prof.rb +++ b/lib/test_prof.rb @@ -1,178 +1,17 @@ # frozen_string_literal: true -require "fileutils" -require "logger" - require "test_prof/version" -require "test_prof/logging" -require "test_prof/utils" - -# Ruby applications tests profiling tools. -# -# Contains tools to analyze factories usage, integrate with Ruby profilers, -# profile your examples using ActiveSupport notifications (if any) and -# statically analyze your code with custom RuboCop cops. -# -# Example usage: -# -# require 'test_prof' -# -# # Activate a tool by providing environment variable, e.g. -# TEST_RUBY_PROF=1 rspec ... -# -# # or manually in your code -# TestProf::RubyProf.run -# -# See other modules for more examples. -module TestProf - class << self - include Logging - - def config - @config ||= Configuration.new - end - - def configure - yield config - end - - # Returns true if we're inside RSpec - def rspec? - defined?(RSpec::Core) - end - - # Returns true if we're inside Minitest - def minitest? - defined?(Minitest) - end - - # Returns true if Spring is used and not disabled - def spring? - # See https://github.com/rails/spring/blob/577cf01f232bb6dbd0ade7df2df2ac209697e741/lib/spring/binstub.rb - disabled = ENV["DISABLE_SPRING"] - defined?(::Spring::Application) && (disabled.nil? || disabled.empty? || disabled == "0") - end - - # Returns the current process time - def now - Process.clock_gettime(Process::CLOCK_MONOTONIC) - end - - # Require gem and shows a custom - # message if it fails to load - def require(gem_name, msg = nil) - Kernel.require gem_name - block_given? ? yield : true - rescue LoadError - log(:error, msg) if msg - false - end - - # Run block only if provided env var is present and - # equal to the provided value (if any). - # Contains workaround for applications using Spring. - def activate(env_var, val = nil) - if spring? - notify_spring_detected - ::Spring.after_fork do - activate!(env_var, val) do - notify_spring_activate env_var - yield - end - end - else - activate!(env_var, val) { yield } - end - end - - # Return absolute path to asset - def asset_path(filename) - ::File.expand_path(filename, ::File.join(::File.dirname(__FILE__), "..", "assets")) - end - - # Return a path to store artifact - def artifact_path(filename) - create_artifact_dir - - with_timestamps( - ::File.join( - config.output_dir, - with_report_suffix( - filename - ) - ) - ) - end - - def create_artifact_dir - FileUtils.mkdir_p(config.output_dir)[0] - end - - private - - def activate!(env_var, val) - yield if ENV[env_var] && (val.nil? || val === ENV[env_var]) - end - - def with_timestamps(path) - return path unless config.timestamps? - timestamps = "-#{now.to_i}" - "#{path.sub(/\.\w+$/, "")}#{timestamps}#{::File.extname(path)}" - end - - def with_report_suffix(path) - return path if config.report_suffix.nil? - - "#{path.sub(/\.\w+$/, "")}-#{config.report_suffix}#{::File.extname(path)}" - end - - def notify_spring_detected - return if instance_variable_defined?(:@spring_notified) - log :info, "Spring detected" - @spring_notified = true - end - - def notify_spring_activate(env_var) - log :info, "Activating #{env_var} with `Spring.after_fork`" - end - end - - # TestProf configuration - class Configuration - attr_accessor :output, # IO to write logs - :color, # Whether to colorize output or not - :output_dir, # Directory to store artifacts - :timestamps, # Whether to use timestamped names for artifacts, - :report_suffix # Custom suffix for reports/artifacts - - def initialize - @output = $stdout - @color = true - @output_dir = "tmp/test_prof" - @timestamps = false - @report_suffix = ENV["TEST_PROF_REPORT"] - end - - def color? - color == true && output.is_a?(IO) && output.tty? - end - - def timestamps? - timestamps == true - end - - def logger - @logger ||= Logger.new(output, formatter: Logging::Formatter.new) - end - end -end +require "test_prof/core" require "test_prof/ruby_prof" require "test_prof/stack_prof" +require "test_prof/vernier" require "test_prof/event_prof" require "test_prof/factory_doctor" require "test_prof/factory_prof" +require "test_prof/memory_prof" require "test_prof/rspec_stamp" require "test_prof/tag_prof" +require "test_prof/tps_prof" require "test_prof/rspec_dissect" if TestProf.rspec? require "test_prof/factory_all_stub" diff --git a/lib/test_prof/any_fixture.rb b/lib/test_prof/any_fixture.rb index 2f724903..f486cff1 100644 --- a/lib/test_prof/any_fixture.rb +++ b/lib/test_prof/any_fixture.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "test_prof/core" require "test_prof/ext/float_duration" require "test_prof/any_fixture/dump" @@ -101,31 +102,24 @@ def configure yield config end - # Backward compatibility - def reporting_enabled=(val) - warn "AnyFixture.reporting_enabled is deprecated and will be removed in 1.1. Use AnyFixture.config.reporting_enabled instead" - config.reporting_enabled = val - end - - def reporting_enabled - warn "AnyFixture.reporting_enabled is deprecated and will be removed in 1.1. Use AnyFixture.config.reporting_enabled instead" - config.reporting_enabled - end - - alias_method :reporting_enabled?, :reporting_enabled - # Register a block of code as a fixture, # returns the result of the block execution def register(id) cached(id) do + raise "No fixture named #{id} has been registered" unless block_given? + + next yield if @disabled + ActiveSupport::Notifications.subscribed(method(:subscriber), "sql.active_record") do yield end end end - def cached(id) - cache.fetch(id) { yield } + def cached(id, &block) + return (block_given? ? yield : nil) if @disabled + + cache.fetch(id, &block) end # Create and register new SQL dump. @@ -174,9 +168,30 @@ def clean # Reset all information and clean tables def reset + callbacks[:before_fixtures_reset].each(&:call) + clean tables_cache.clear cache.clear + + callbacks[:after_fixtures_reset].each(&:call) + callbacks.clear + end + + def disable! + @disabled = true + end + + def enable! + @disabled = false + end + + def before_fixtures_reset(&block) + callbacks[:before_fixtures_reset] << block + end + + def after_fixtures_reset(&block) + callbacks[:after_fixtures_reset] << block end def subscriber(_event, _start, _finish, _id, data) @@ -253,11 +268,17 @@ def tables_cache @tables_cache ||= {} end + def callbacks + @callbacks ||= Hash.new { |h, k| h[k] = [] } + end + def disable_referential_integrity connection = ActiveRecord::Base.connection return yield unless connection.respond_to?(:disable_referential_integrity) connection.disable_referential_integrity { yield } end end + + enable! end end diff --git a/lib/test_prof/any_fixture/dsl.rb b/lib/test_prof/any_fixture/dsl.rb index dbaa63cf..60ca7cc6 100644 --- a/lib/test_prof/any_fixture/dsl.rb +++ b/lib/test_prof/any_fixture/dsl.rb @@ -1,15 +1,53 @@ # frozen_string_literal: true +if defined?(::ActiveRecord::Base) + require "test_prof/ext/active_record_refind" + using TestProf::Ext::ActiveRecordRefind +end + module TestProf module AnyFixture - # Adds "global" `fixture` method (through refinement) + # Adds "global" `fixture`, `before_fixtures_reset` and `after_fixtures_reset` methods (through refinement) module DSL + module Methods + def fixture(id, &block) + id = :"#{id}" + record = ::TestProf::AnyFixture.cached(id) + + return ::TestProf::AnyFixture.register(id, &block) unless record + + return record.refind if record.is_a?(::ActiveRecord::Base) + + if record.respond_to?(:to_ary) + return record.map do |rec| + rec.is_a?(::ActiveRecord::Base) ? rec.refind : rec + end + end + + record + end + + def before_fixtures_reset(&block) + ::TestProf::AnyFixture.before_fixtures_reset(&block) + end + + def after_fixtures_reset(&block) + ::TestProf::AnyFixture.after_fixtures_reset(&block) + end + end + + def self.included(base) + base.include Methods + end + # Refine object, 'cause refining modules (Kernel) is vulnerable to prepend: # - https://bugs.ruby-lang.org/issues/13446 # - Rails added `Kernel.prepend` in 6.1: https://github.com/rails/rails/commit/3124007bd674dcdc9c3b5c6b2964dfb7a1a0733c refine ::Object do - def fixture(id, &block) - ::TestProf::AnyFixture.register(:"#{id}", &block) + if RUBY_VERSION >= "3.1.0" + import_methods Methods + else + include Methods end end end diff --git a/lib/test_prof/any_fixture/dump.rb b/lib/test_prof/any_fixture/dump.rb index 7dc037ac..82d33cc2 100644 --- a/lib/test_prof/any_fixture/dump.rb +++ b/lib/test_prof/any_fixture/dump.rb @@ -69,7 +69,7 @@ def finish(_event, _id, payload) end def commit - return unless defined?(:@file) + return unless instance_variable_defined?(:@file) file.close diff --git a/lib/test_prof/any_fixture/dump/postgresql.rb b/lib/test_prof/any_fixture/dump/postgresql.rb index 5a22841d..4e42989f 100644 --- a/lib/test_prof/any_fixture/dump/postgresql.rb +++ b/lib/test_prof/any_fixture/dump/postgresql.rb @@ -23,14 +23,14 @@ def reset_sequence!(table_name, start) end def compile_sql(sql, binds) - sql.gsub(/\$\d+/) { binds.shift } + sql.gsub(/\$\d+/) { binds.shift.gsub("\n", "' || chr(10) || '") } end def import(path) # Test if psql is installed `psql --version` - tasks = ActiveRecord::Tasks::PostgreSQLDatabaseTasks.new(conn.pool.spec.config.with_indifferent_access) + tasks = ActiveRecord::Tasks::PostgreSQLDatabaseTasks.new(config) while_disconnected do tasks.structure_load(path, "--output=/dev/null") @@ -85,6 +85,15 @@ def teardown_env def execute(query) super.values end + + def config + conn_pool = conn.pool + if conn_pool.respond_to?(:spec) # Support for Rails < 6.1 + conn_pool.spec.config + else + conn_pool.db_config + end + end end end end diff --git a/lib/test_prof/any_fixture/dump/sqlite.rb b/lib/test_prof/any_fixture/dump/sqlite.rb index 5c6c9357..962af315 100644 --- a/lib/test_prof/any_fixture/dump/sqlite.rb +++ b/lib/test_prof/any_fixture/dump/sqlite.rb @@ -18,7 +18,7 @@ def reset_sequence!(table_name, start) end def compile_sql(sql, binds) - sql.gsub(/\?/) { binds.shift } + sql.gsub("?") { binds.shift.gsub("\n", "' || char(10) || '") } end def import(path) diff --git a/lib/test_prof/before_all.rb b/lib/test_prof/before_all.rb index b654a353..0d148067 100644 --- a/lib/test_prof/before_all.rb +++ b/lib/test_prof/before_all.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "test_prof/core" + module TestProf # `before_all` helper configuration module BeforeAll @@ -12,22 +14,40 @@ def initialize end end + # Used in dry-run mode + class NoopAdapter + class << self + def begin_transaction(...) + end + + def rollback_transaction(...) + end + + def setup_fixtures(...) + end + end + end + class << self - attr_accessor :adapter + attr_writer :adapter + + def adapter + @adapter ||= default_adapter + end - def begin_transaction + def begin_transaction(scope = nil, metadata = []) raise AdapterMissing if adapter.nil? - config.run_hooks(:begin) do + config.run_hooks(:begin, scope, metadata) do adapter.begin_transaction end yield end - def rollback_transaction + def rollback_transaction(scope = nil, metadata = []) raise AdapterMissing if adapter.nil? - config.run_hooks(:rollback) do + config.run_hooks(:rollback, scope, metadata) do adapter.rollback_transaction end end @@ -45,6 +65,44 @@ def config def configure yield config end + + private + + def default_adapter + return NoopAdapter if TestProf.dry_run? + + if defined?(::ActiveRecord::Base) + require "test_prof/before_all/adapters/active_record" + Adapters::ActiveRecord + end + end + end + + class HookEntry # :nodoc: + attr_reader :filters, :block + + def initialize(block:, filters: []) + @block = block + @filters = TestProf.rspec? ? ::RSpec::Core::Metadata.build_hash_from(filters) : filters + end + + def run(scope, metadata) + return unless filters_apply?(metadata) + + block.call(scope) + end + + private + + def filters_apply?(metadata) + return true unless filters.is_a?(Hash) && TestProf.rspec? + + ::RSpec::Core::MetadataFilter.apply?( + :all?, + filters, + metadata + ) + end end class HooksChain # :nodoc: @@ -56,10 +114,10 @@ def initialize(type) @after = [] end - def run - before.each(&:call) + def run(scope = nil, metadata = []) + before.each { |hook| hook.run(scope, metadata) } yield - after.each(&:call) + after.each { |hook| hook.run(scope, metadata) } end end @@ -74,26 +132,26 @@ def initialize end # Add `before` hook for `begin` or - # `rollback` operation: + # `rollback` operation with optional filters: # - # config.before(:rollback) { ... } - def before(type, &block) + # config.before(:rollback, foo: :bar) { ... } + def before(type, *filters, &block) validate_hook_type!(type) - hooks[type].before << block if block + hooks[type].before << HookEntry.new(block: block, filters: filters) if block end # Add `after` hook for `begin` or - # `rollback` operation: + # `rollback` operation with optional filters: # - # config.after(:begin) { ... } - def after(type, &block) + # config.after(:begin, foo: :bar) { ... } + def after(type, *filters, &block) validate_hook_type!(type) - hooks[type].after << block if block + hooks[type].after << HookEntry.new(block: block, filters: filters) if block end - def run_hooks(type) # :nodoc: + def run_hooks(type, scope = nil, metadata = []) # :nodoc: validate_hook_type!(type) - hooks[type].run { yield } + hooks[type].run(scope, metadata) { yield } end private @@ -109,12 +167,6 @@ def validate_hook_type!(type) end end -if defined?(::ActiveRecord::Base) - require "test_prof/before_all/adapters/active_record" - - TestProf::BeforeAll.adapter = TestProf::BeforeAll::Adapters::ActiveRecord -end - if defined?(::Isolator) require "test_prof/before_all/isolator" end diff --git a/lib/test_prof/before_all/adapters/active_record.rb b/lib/test_prof/before_all/adapters/active_record.rb index 48c96444..2a94ed59 100644 --- a/lib/test_prof/before_all/adapters/active_record.rb +++ b/lib/test_prof/before_all/adapters/active_record.rb @@ -5,18 +5,93 @@ module BeforeAll module Adapters # ActiveRecord adapter for `before_all` module ActiveRecord + POOL_ARGS = ((::ActiveRecord::VERSION::MAJOR > 6) ? [:writing] : []).freeze + class << self - def begin_transaction - ::ActiveRecord::Base.connection.begin_transaction(joinable: false) - end + if ::ActiveRecord::Base.connection.pool.respond_to?(:pin_connection!) + def begin_transaction + subscribe! + ::ActiveRecord::Base.connection_handler.connection_pool_list(:writing).each do |pool| + pool.pin_connection!(true) + end + end + + def rollback_transaction + ::ActiveRecord::Base.connection_handler.connection_pool_list(:writing).each do |pool| + pool.unpin_connection! + end + unsubscribe! + end + + def subscribe! + Thread.current[:before_all_subscription_count] ||= 0 + Thread.current[:before_all_subscription_count] += 1 + + return unless Thread.current[:before_all_subscription_count] == 1 + + Thread.current[:before_all_connection_subscriber] = ActiveSupport::Notifications.subscribe("!connection.active_record") do |_, _, _, _, payload| + connection_name = payload[:connection_name] if payload.key?(:connection_name) + shard = payload[:shard] if payload.key?(:shard) + next unless connection_name + + pool = ::ActiveRecord::Base.connection_handler.retrieve_connection_pool(connection_name, shard: shard) + next unless pool && pool.role == :writing + + pool.pin_connection!(true) + end + end + + def unsubscribe! + return unless Thread.current[:before_all_subscription_count] + + Thread.current[:before_all_subscription_count] -= 1 + + return unless Thread.current[:before_all_subscription_count] == 0 && Thread.current[:before_all_connection_subscriber] - def rollback_transaction - if ::ActiveRecord::Base.connection.open_transactions.zero? - warn "!!! before_all transaction has been already rollbacked and " \ - "could work incorrectly" - return + ActiveSupport::Notifications.unsubscribe(Thread.current[:before_all_connection_subscriber]) + Thread.current[:before_all_connection_subscriber] = nil + end + else + def all_connections + @all_connections ||= if ::ActiveRecord::Base.respond_to? :connects_to + ::ActiveRecord::Base.connection_handler.connection_pool_list(*POOL_ARGS).filter_map { |pool| + begin + pool.connection + rescue *pool_connection_errors => error + log_pool_connection_error(pool, error) + nil + end + } + else + Array.wrap(::ActiveRecord::Base.connection) + end + end + + def pool_connection_errors + @pool_connection_errors ||= [] + end + + def log_pool_connection_error(pool, error) + warn "Could not connect to pool #{pool.connection_class.name}. #{error.class}: #{error.message}" + end + + def begin_transaction + @all_connections = nil + all_connections.each do |connection| + connection.begin_transaction(joinable: false) + end + end + + def rollback_transaction + all_connections.each do |connection| + if connection.open_transactions.zero? + warn "!!! before_all transaction has been already rollbacked and " \ + "could work incorrectly" + next + end + connection.rollback_transaction + end end - ::ActiveRecord::Base.connection.rollback_transaction end def setup_fixtures(test_object) @@ -37,14 +112,23 @@ def setup_fixtures(test_object) end end - configure do |config| - # Make sure ActiveRecord uses locked thread. - # It only gets locked in `before` / `setup` hook, - # thus using thread in `before_all` (e.g. ActiveJob async adapter) - # might lead to leaking connections - config.before(:begin) do - next unless ::ActiveRecord::Base.connection.pool.respond_to?(:lock_thread=) - ::ActiveRecord::Base.connection.pool.lock_thread = true + unless ::ActiveRecord::Base.connection.pool.respond_to?(:pin_connection!) + # avoid instance variable collisions with cats + PREFIX_RESTORE_LOCK_THREAD = "@😺" + + configure do |config| + # Make sure ActiveRecord uses locked thread. + # It only gets locked in `before` / `setup` hook, + # thus using thread in `before_all` (e.g. ActiveJob async adapter) + # might lead to leaking connections + config.before(:begin) do + instance_variable_set("#{PREFIX_RESTORE_LOCK_THREAD}_orig_lock_thread", ::ActiveRecord::Base.connection.pool.instance_variable_get(:@lock_thread)) unless instance_variable_defined? "#{PREFIX_RESTORE_LOCK_THREAD}_orig_lock_thread" + ::ActiveRecord::Base.connection.pool.lock_thread = true + end + + config.after(:rollback) do + ::ActiveRecord::Base.connection.pool.lock_thread = instance_variable_get("#{PREFIX_RESTORE_LOCK_THREAD}_orig_lock_thread") + end end end end diff --git a/lib/test_prof/before_all/isolator.rb b/lib/test_prof/before_all/isolator.rb index f6a27fdc..66e7d827 100644 --- a/lib/test_prof/before_all/isolator.rb +++ b/lib/test_prof/before_all/isolator.rb @@ -1,20 +1,11 @@ # frozen_string_literal: true -module TestProf - module BeforeAll - # Disable Isolator within before_all blocks - module Isolator - def begin_transaction(*) - ::Isolator.transactions_threshold += 1 - super - end +TestProf::BeforeAll.configure do |config| + config.before(:begin) do + ::Isolator.incr_thresholds! + end - def rollback_transaction(*) - super - ::Isolator.transactions_threshold -= 1 - end - end + config.after(:rollback) do + ::Isolator.decr_thresholds! end end - -TestProf::BeforeAll.singleton_class.prepend(TestProf::BeforeAll::Isolator) diff --git a/lib/test_prof/cops/inject.rb b/lib/test_prof/cops/inject.rb deleted file mode 100644 index de671090..00000000 --- a/lib/test_prof/cops/inject.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -# This is shamelessly borrowed from RuboCop RSpec -# https://github.com/rubocop-hq/rubocop-rspec/blob/master/lib/rubocop/rspec/inject.rb -module TestProf - module Cops - # Because RuboCop doesn't yet support plugins, we have to monkey patch in a - # bit of our configuration. - module Inject - PROJECT_ROOT = Pathname.new(__dir__).parent.parent.parent.expand_path.freeze - CONFIG_DEFAULT = PROJECT_ROOT.join("config", "default.yml").freeze - - def self.defaults! - path = CONFIG_DEFAULT.to_s - hash = RuboCop::ConfigLoader.send(:load_yaml_configuration, path) - config = RuboCop::Config.new(hash, path) - puts "configuration from #{path}" if RuboCop::ConfigLoader.debug? - config = RuboCop::ConfigLoader.merge_with_default(config, path) - RuboCop::ConfigLoader.instance_variable_set(:@default_configuration, config) - end - end - end -end - -TestProf::Cops::Inject.defaults! diff --git a/lib/test_prof/core.rb b/lib/test_prof/core.rb new file mode 100644 index 00000000..4c258e67 --- /dev/null +++ b/lib/test_prof/core.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +require "fileutils" +require "logger" + +require "test_prof/logging" +require "test_prof/utils" + +# Add an alias for Process.clock_gettime "reserved" for TestProf +# (in case some other tool would like to patch it) +module ::Process + class << self + # Already patched by Timecop + if method_defined?(:clock_gettime_without_mock) + alias_method :clock_gettime_for_test_prof, :clock_gettime_without_mock + else + alias_method :clock_gettime_for_test_prof, :clock_gettime + + def singleton_method_added(method_name) + return super unless method_name == :clock_gettime_without_mock + + define_method(:clock_gettime_for_test_prof) { |*args| clock_gettime_without_mock(*args) } + end + end + end +end + +# Main TestProf module +# +# Contains configuration and common methods + +# Ruby applications tests profiling tools. +# +# Contains tools to analyze factories usage, integrate with Ruby profilers, +# profile your examples using ActiveSupport notifications (if any) and +# statically analyze your code with custom RuboCop cops. +# +# Example usage: +# +# require 'test_prof' +# +# # Activate a tool by providing environment variable, e.g. +# TEST_RUBY_PROF=1 rspec ... +# +# # or manually in your code +# TestProf::RubyProf.run +# +# See other modules for more examples. +module TestProf + class << self + include Logging + + def config + @config ||= Configuration.new + end + + def configure + yield config + end + + # Returns true if we're inside RSpec + def rspec? + defined?(RSpec::Core) + end + + # Returns true if we're inside Minitest + def minitest? + defined?(Minitest) + end + + # Returns true if Spring is used and not disabled + def spring? + # See https://github.com/rails/spring/blob/577cf01f232bb6dbd0ade7df2df2ac209697e741/lib/spring/binstub.rb + disabled = ENV["DISABLE_SPRING"] + defined?(::Spring::Application) && (disabled.nil? || disabled.empty? || disabled == "0") + end + + def dry_run? + rspec? && ::RSpec.configuration.dry_run? + end + + # Returns the current process time + def now + Process.clock_gettime_for_test_prof(Process::CLOCK_MONOTONIC) + end + + # Require gem and shows a custom + # message if it fails to load + def require(gem_name, msg = nil) + Kernel.require gem_name + block_given? ? yield : true + rescue LoadError + log(:error, msg) if msg + false + end + + # Run block only if provided env var is present and + # equal to the provided value (if any). + # Contains workaround for applications using Spring. + def activate(env_var, val = nil) + if spring? + notify_spring_detected + ::Spring.after_fork do + activate!(env_var, val) do + notify_spring_activate env_var + yield + end + end + else + activate!(env_var, val) { yield } + end + end + + # Return absolute path to asset + def asset_path(filename) + ::File.expand_path(filename, ::File.join(::File.dirname(__FILE__), "..", "..", "assets")) + end + + # Return a path to store artifact + def artifact_path(filename) + create_artifact_dir + + with_timestamps( + ::File.join( + config.output_dir, + with_report_suffix( + filename + ) + ) + ) + end + + def create_artifact_dir + FileUtils.mkdir_p(config.output_dir)[0] + end + + private + + def activate!(env_var, val) + yield if ENV[env_var] && (val.nil? || val === ENV[env_var]) + end + + def with_timestamps(path) + return path unless config.timestamps? + timestamps = "-#{now.to_i}" + "#{path.sub(/\.\w+$/, "")}#{timestamps}#{::File.extname(path)}" + end + + def with_report_suffix(path) + return path if config.report_suffix.nil? + + "#{path.sub(/\.\w+$/, "")}-#{config.report_suffix}#{::File.extname(path)}" + end + + def notify_spring_detected + return if instance_variable_defined?(:@spring_notified) + log :info, "Spring detected" + @spring_notified = true + end + + def notify_spring_activate(env_var) + log :info, "Activating #{env_var} with `Spring.after_fork`" + end + end + + # TestProf configuration + class Configuration + attr_accessor :output, # IO to write logs + :color, # Whether to colorize output or not + :output_dir, # Directory to store artifacts + :timestamps, # Whether to use timestamped names for artifacts, + :report_suffix # Custom suffix for reports/artifacts + + def initialize + @output = $stdout + @color = true + @output_dir = "tmp/test_prof" + @timestamps = false + @report_suffix = ENV["TEST_PROF_REPORT"] + end + + def color? + color == true && output.is_a?(IO) && output.tty? + end + + def timestamps? + timestamps == true + end + + def logger + @logger ||= Logger.new(output, formatter: Logging::Formatter.new) + end + end +end diff --git a/lib/test_prof/event_prof.rb b/lib/test_prof/event_prof.rb index 6eea925a..68e29413 100644 --- a/lib/test_prof/event_prof.rb +++ b/lib/test_prof/event_prof.rb @@ -8,7 +8,7 @@ module TestProf # EventProf profiles your tests and suites against custom events, - # such as ActiveSupport::Notifacations. + # such as ActiveSupport::Notifications. # # It works very similar to `rspec --profile` but can track arbitrary events. # diff --git a/lib/test_prof/event_prof/custom_events/factory_create.rb b/lib/test_prof/event_prof/custom_events/factory_create.rb index f7ffcef7..04636ef0 100644 --- a/lib/test_prof/event_prof/custom_events/factory_create.rb +++ b/lib/test_prof/event_prof/custom_events/factory_create.rb @@ -29,7 +29,7 @@ TestProf.log( :error, <<~MSG - Failed to load factory_bot / factory_girl / fabrication. + Failed to load factory_bot / fabrication. Make sure that any of them is in your Gemfile. MSG diff --git a/lib/test_prof/event_prof/monitor.rb b/lib/test_prof/event_prof/monitor.rb index c0a599d4..3a811670 100644 --- a/lib/test_prof/event_prof/monitor.rb +++ b/lib/test_prof/event_prof/monitor.rb @@ -48,9 +48,9 @@ def call(mod, event, *mids, guard: nil, top_level: false) patch = Module.new do mids.each do |mid| - define_method(mid) do |*args, &block| - next super(*args, &block) unless guard.nil? || instance_exec(*args, &guard) - tracker.track { super(*args, &block) } + define_method(mid) do |*args, **kwargs, &block| + next super(*args, **kwargs, &block) unless guard.nil? || instance_exec(*args, **kwargs, &guard) + tracker.track { super(*args, **kwargs, &block) } end end end diff --git a/lib/test_prof/event_prof/rspec.rb b/lib/test_prof/event_prof/rspec.rb index 1b3d24f9..d661d4d0 100644 --- a/lib/test_prof/event_prof/rspec.rb +++ b/lib/test_prof/event_prof/rspec.rb @@ -7,6 +7,7 @@ module TestProf module EventProf class RSpecListener # :nodoc: include Logging + using FloatDuration using StringTruncate diff --git a/lib/test_prof/ext/active_record_refind.rb b/lib/test_prof/ext/active_record_refind.rb index fbc2d9cc..d1ff4970 100644 --- a/lib/test_prof/ext/active_record_refind.rb +++ b/lib/test_prof/ext/active_record_refind.rb @@ -12,7 +12,7 @@ module ActiveRecordRefind # # We need it to make sure that the state is clean. def refind - self.class.find(send(self.class.primary_key)) + self.class.unscoped.find(send(self.class.primary_key)) end end end diff --git a/lib/test_prof/ext/string_truncate.rb b/lib/test_prof/ext/string_truncate.rb index 7972e438..2953be88 100644 --- a/lib/test_prof/ext/string_truncate.rb +++ b/lib/test_prof/ext/string_truncate.rb @@ -12,7 +12,7 @@ def truncate(limit = 30) head = ((limit - 3) / 2) tail = head + 3 - limit - "#{self[0..(head - 1)]}...#{self[tail..-1]}" + "#{self[0..(head - 1)]}...#{self[tail..]}" end end end diff --git a/lib/test_prof/factory_all_stub.rb b/lib/test_prof/factory_all_stub.rb index 06b33b57..4a116738 100644 --- a/lib/test_prof/factory_all_stub.rb +++ b/lib/test_prof/factory_all_stub.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "test_prof/core" require "test_prof/factory_bot" require "test_prof/factory_all_stub/factory_bot_patch" @@ -11,7 +12,7 @@ module FactoryAllStub class << self def init - # Monkey-patch FactoryBot / FactoryGirl + # Monkey-patch FactoryBot TestProf::FactoryBot::FactoryRunner.prepend(FactoryBotPatch) if defined?(TestProf::FactoryBot) end diff --git a/lib/test_prof/factory_bot.rb b/lib/test_prof/factory_bot.rb index 2a6946aa..20d0461d 100644 --- a/lib/test_prof/factory_bot.rb +++ b/lib/test_prof/factory_bot.rb @@ -1,11 +1,9 @@ # frozen_string_literal: true module TestProf # :nodoc: all - FACTORY_GIRL_NAMES = {"factory_bot" => "::FactoryBot", "factory_girl" => "::FactoryGirl"}.freeze + TestProf.require("active_support") - FACTORY_GIRL_NAMES.find do |name, cname| - TestProf.require(name) do - TestProf::FactoryBot = Object.const_get(cname) - end + TestProf.require("factory_bot") do + TestProf::FactoryBot = Object.const_get("::FactoryBot") end end diff --git a/lib/test_prof/factory_default.rb b/lib/test_prof/factory_default.rb index f4efdb93..4ade23c7 100644 --- a/lib/test_prof/factory_default.rb +++ b/lib/test_prof/factory_default.rb @@ -1,68 +1,356 @@ # frozen_string_literal: true -require "test_prof/factory_bot" +require "test_prof/core" + require "test_prof/factory_default/factory_bot_patch" +require "test_prof/factory_default/fabrication_patch" + +require "test_prof/ext/float_duration" +require "test_prof/ext/active_record_refind" if defined?(::ActiveRecord::Base) module TestProf # FactoryDefault allows use to re-use associated objects # in factories implicilty module FactoryDefault - module DefaultSyntax # :nodoc: - def create_default(name, *args, &block) - options = args.extract_options! - preserve = options.delete(:preserve_traits) + using FloatDuration + using Ext::ActiveRecordRefind if defined?(::ActiveRecord::Base) + + using(Module.new do + refine Object do + def to_override_key + "<#{self.class.name}::$id$#{object_id}$di$>" + end + + def refind + self + end + end - obj = TestProf::FactoryBot.create(name, *args, options, &block) - set_factory_default(name, obj, preserve_traits: preserve) + if defined?(::ActiveRecord::Base) + refine ::ActiveRecord::Base do + def to_override_key + "<#{self.class.name}\#$id$#{public_send(self.class.primary_key)}$di$>" + end + end end - def set_factory_default(name, obj, preserve_traits: nil) - FactoryDefault.register(name, obj, preserve_traits: preserve_traits) + [ + String, + Integer, + Float, + FalseClass, + TrueClass, + NilClass, + Regexp + ].each do |mod| + refine(mod) do + def to_override_key + inspect + end + end + end + end) + + class Profiler + include Logging + + attr_reader :data + + def initialize + @data = Hash.new { |h, k| h[k] = {count: 0, time: 0.0} } + end + + def instrument(name, traits, overrides) + start = TestProf.now + yield.tap do + time = TestProf.now - start + key = build_association_name(name, traits, overrides) + data[key][:count] += 1 + data[key][:time] += time + end + end + + def print_report + if data.empty? + log :info, "FactoryDefault profiler collected no data" + return + end + + # Merge object overrides into one stats record + data = self.data.each_with_object({}) do |(name, stats), acc| + name = name.gsub(/\$id\$.+\$di\$/, "") + if acc.key?(name) + acc[name][:count] += stats[:count] + acc[name][:time] += stats[:time] + else + acc[name] = stats + end + end + + msgs = [] + + msgs << + <<~MSG + Factory associations usage: + MSG + + first_column = data.keys.map(&:size).max + 2 + + msgs << format( + "%#{first_column}s %9s %12s", + "factory", "count", "total time" + ) + + msgs << "" + + total_count = 0 + total_time = 0.0 + + data.to_a.sort_by { |(_, v)| -v[:time] }.each do |(key, factory_stats)| + total_count += factory_stats[:count] + total_time += factory_stats[:time] + + msgs << format( + "%#{first_column}s %9d %12s", + key, factory_stats[:count], factory_stats[:time].duration + ) + end + + msgs << + <<~MSG + + Total associations created: #{total_count} + Total uniq associations created: #{data.size} + Total time spent: #{total_time.duration} + + MSG + + log :info, msgs.join("\n") + end + + private + + def build_association_name(name, traits, overrides) + traits_str = "[#{traits.join(",")}]" if traits&.any? + overrides_str = "{#{overrides.map { |k, v| "#{k}:#{v.to_override_key}" }.join(",")}}" if overrides&.any? + "#{name}#{traits_str}#{overrides_str}" + end + end + + class NoopProfiler + def instrument(*) + yield + end + + def print_report + end + end + + class Configuration + attr_accessor :preserve_traits, :preserve_attributes, + :report_summary, :report_stats, + :profiling_enabled + + alias_method :profiling_enabled?, :profiling_enabled + + def initialize + # TODO(v2): Switch to true + @preserve_traits = false + @preserve_attributes = false + @profiling_enabled = ENV["FACTORY_DEFAULT_PROF"] == "1" + @report_summary = ENV["FACTORY_DEFAULT_SUMMARY"] == "1" + @report_stats = ENV["FACTORY_DEFAULT_STATS"] == "1" end end class << self - attr_accessor :preserve_traits + include Logging + + attr_accessor :current_context + attr_reader :stats, :profiler def init - TestProf::FactoryBot::Syntax::Methods.include DefaultSyntax - TestProf::FactoryBot.extend DefaultSyntax - TestProf::FactoryBot::Strategy::Create.prepend StrategyExt - TestProf::FactoryBot::Strategy::Build.prepend StrategyExt - TestProf::FactoryBot::Strategy::Stub.prepend StrategyExt - - @store = {} - # default is false to retain backward compatibility - @preserve_traits = false + FactoryBotPatch.patch + FabricationPatch.patch + + @profiler = config.profiling_enabled? ? Profiler.new : NoopProfiler.new + @enabled = ENV["FACTORY_DEFAULT_DISABLED"] != "1" + @stats = {} + end + + def config + @config ||= Configuration.new + end + + def configure + yield config + end + + # TODO(v2): drop + def preserve_traits=(val) + config.preserve_traits = val + end + + def preserve_attributes=(val) + config.preserve_attributes = val end def register(name, obj, **options) - options[:preserve_traits] = true if FactoryDefault.preserve_traits - store[name] = {object: obj, **options} + # Name with traits + if name.is_a?(Array) + register_traited_record(*name, obj, **options) + else + register_default_record(name, obj, **options) + end + obj end - def get(name, traits = nil) + def get(name, traits = nil, overrides = nil, skip_stats: false) + return unless enabled? + record = store[name] return unless record - if traits && !traits.empty? - return if FactoryDefault.preserve_traits || record[:preserve_traits] + if traits && (trait_key = record[:traits][traits]) + name = trait_key + record = store[name] + traits = nil + end + + stats[name][:miss] += 1 unless skip_stats + + if traits && !traits.empty? && record[:preserve_traits] + return + end + + object = record[:object] + + if overrides && !overrides.empty? && record[:preserve_attributes] + overrides.each do |name, value| + return unless object.respond_to?(name) # rubocop:disable Lint/NonLocalExitFromIterator + return if object.public_send(name) != value # rubocop:disable Lint/NonLocalExitFromIterator + end + end + + unless skip_stats + stats[name][:miss] -= 1 + stats[name][:hit] += 1 + end + + if record[:context] && (record[:context] != :example) + object.refind + else + object end - record[:object] end def remove(name) store.delete(name) end - def reset - @store.clear + def reset(context: nil) + return store.clear unless context + + store.delete_if do |_name, metadata| + metadata[:context] == context + end + end + + def enabled? + @enabled + end + + def enable! + was_enabled = @enabled + @enabled = true + return unless block_given? + yield + ensure + @enabled = was_enabled + end + + def disable! + was_enabled = @enabled + @enabled = false + return unless block_given? + yield + ensure + @enabled = was_enabled + end + + def print_report + profiler.print_report + return unless config.report_stats || config.report_summary + + if stats.empty? + log :info, "FactoryDefault has not been used" + return + end + + msgs = [] + + if config.report_stats + msgs << + <<~MSG + FactoryDefault usage stats: + MSG + + first_column = stats.keys.map(&:size).max + 2 + + msgs << format( + "%#{first_column}s %9s %9s", + "factory", "hit", "miss" + ) + + msgs << "" + end + + total_hit = 0 + total_miss = 0 + + stats.to_a.sort_by { |(_, v)| -v[:hit] }.each do |(key, record_stats)| + total_hit += record_stats[:hit] + total_miss += record_stats[:miss] + + if config.report_stats + msgs << format( + "%#{first_column}s %9d %9d", + key, record_stats[:hit], record_stats[:miss] + ) + end + end + + msgs << "" if config.report_stats + + msgs << + <<~MSG + FactoryDefault summary: hit=#{total_hit} miss=#{total_miss} + MSG + + log :info, msgs.join("\n") end private - attr_reader :store + def register_default_record(name, obj, **options) + store[name] = {object: obj, traits: {}, context: current_context, **options} + stats[name] ||= {hit: 0, miss: 0} + end + + def register_traited_record(name, *traits, obj, **options) + name_with_traits = "#{name}[#{traits.join(",")}]" + + register_default_record(name_with_traits, obj, **options) + register_default_record(name, obj, **options) unless store[name] + + # Add reference to the traited default to the original default record + store[name][:traits][traits] = name_with_traits + end + + def store + Thread.current[:testprof_factory_default_store] ||= {} + end end end end diff --git a/lib/test_prof/factory_default/fabrication_patch.rb b/lib/test_prof/factory_default/fabrication_patch.rb new file mode 100644 index 00000000..d55cf0e4 --- /dev/null +++ b/lib/test_prof/factory_default/fabrication_patch.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module TestProf + module FactoryDefault # :nodoc: all + module FabricationPatch + module DefaultExt + def create_default(name, overrides = {}, &block) + obj = ::Fabricate.create(name, overrides, &block) + set_fabricate_default(name, obj) + end + + def set_fabricate_default(name, obj, **opts) + FactoryDefault.register( + name, obj, + preserve_attributes: FactoryDefault.config.preserve_attributes, + preserve_traits: FactoryDefault.config.preserve_traits, + **opts + ) + end + + def get_fabricate_default(name, **overrides) + FactoryDefault.get(name, nil, overrides, skip_stats: true) + end + + def skip_fabricate_default(&block) + FactoryDefault.disable!(&block) + end + + def create(name, overrides = {}, &block) + self.fabrication_depth += 1 + # We do not support defaults for objects created with attribute blocks + return super if block + + return super if fabrication_depth < 2 + + FactoryDefault.get(name, nil, overrides, **{}) || + FactoryDefault.profiler.instrument(name, nil, overrides) { super } + ensure + self.fabrication_depth -= 1 + end + + private + + def fabrication_depth + Thread.current[:_fab_depth_] ||= 0 + end + + def fabrication_depth=(value) + Thread.current[:_fab_depth_] = value + end + end + + def self.patch + TestProf.require "fabrication" do + ::Fabricate.singleton_class.prepend(DefaultExt) + end + end + end + end +end diff --git a/lib/test_prof/factory_default/factory_bot_patch.rb b/lib/test_prof/factory_default/factory_bot_patch.rb index 065d8c59..b185b2bc 100644 --- a/lib/test_prof/factory_default/factory_bot_patch.rb +++ b/lib/test_prof/factory_default/factory_bot_patch.rb @@ -1,24 +1,69 @@ # frozen_string_literal: true +require "test_prof/factory_bot" + module TestProf module FactoryDefault # :nodoc: all - module RunnerExt - refine TestProf::FactoryBot::FactoryRunner do - def name - @name + module FactoryBotPatch + if defined?(TestProf::FactoryBot::FactoryRunner) + module RunnerExt + refine TestProf::FactoryBot::FactoryRunner do + attr_reader :name, :traits, :overrides + end end - def traits - @traits + using RunnerExt + end + + module StrategyExt + def association(runner) + FactoryDefault.get(runner.name, runner.traits, runner.overrides, **{}) || + FactoryDefault.profiler.instrument(runner.name, runner.traits, runner.overrides) { super } + end + end + + module SyntaxExt + def create_default(name, *args, &block) + options = args.extract_options! + default_options = {} + default_options[:preserve_traits] = options.delete(:preserve_traits) if options.key?(:preserve_traits) + default_options[:preserve_attributes] = options.delete(:preserve_attributes) if options.key?(:preserve_attributes) + + obj = TestProf::FactoryBot.create(name, *args, options, &block) + + # Factory with traits + name = [name, *args] if args.any? + + set_factory_default(name, obj, **default_options) + end + + def set_factory_default(*name, obj, preserve_traits: FactoryDefault.config.preserve_traits, preserve_attributes: FactoryDefault.config.preserve_attributes, **other) + name = name.first if name.size == 1 + FactoryDefault.register( + name, obj, + preserve_traits: preserve_traits, + preserve_attributes: preserve_attributes, + **other + ) + end + + def get_factory_default(name, *traits, **overrides) + FactoryDefault.get(name, traits, overrides, skip_stats: true) + end + + def skip_factory_default(&block) + FactoryDefault.disable!(&block) end end - end - using RunnerExt + def self.patch + return unless defined?(TestProf::FactoryBot) - module StrategyExt - def association(runner) - FactoryDefault.get(runner.name, runner.traits) || super + TestProf::FactoryBot::Syntax::Methods.include SyntaxExt + TestProf::FactoryBot.extend SyntaxExt + TestProf::FactoryBot::Strategy::Create.prepend StrategyExt + TestProf::FactoryBot::Strategy::Build.prepend StrategyExt + TestProf::FactoryBot::Strategy::Stub.prepend StrategyExt end end end diff --git a/lib/test_prof/factory_doctor.rb b/lib/test_prof/factory_doctor.rb index 9c564f28..5b99fce7 100644 --- a/lib/test_prof/factory_doctor.rb +++ b/lib/test_prof/factory_doctor.rb @@ -27,6 +27,7 @@ def bad? pg_attribute| pg_namespace| show\stables| + show\ssearch_path| pragma| sqlite_master/rollback| \ATRUNCATE TABLE| @@ -70,7 +71,7 @@ def init log :info, "FactoryDoctor enabled (event: \"#{config.event}\", threshold: #{config.threshold})" - # Monkey-patch FactoryBot / FactoryGirl + # Monkey-patch FactoryBot TestProf::FactoryBot::FactoryRunner.prepend(FactoryBotPatch) if defined?(TestProf::FactoryBot) @@ -105,10 +106,9 @@ def result # Do not analyze code within the block def ignore @ignored = true - res = yield + yield ensure @ignored = false - res end def ignore! diff --git a/lib/test_prof/factory_doctor/minitest.rb b/lib/test_prof/factory_doctor/minitest.rb index bbe6c287..7b1de4d0 100644 --- a/lib/test_prof/factory_doctor/minitest.rb +++ b/lib/test_prof/factory_doctor/minitest.rb @@ -74,8 +74,8 @@ def report @example_groups.each do |group, examples| msgs << "#{group[:description]} (#{group[:location]})\n" examples.each do |ex| - msgs << " #{ex[:description]} (#{ex[:location]}) "\ - "– #{pluralize_records(ex[:factories])} created, "\ + msgs << " #{ex[:description]} (#{ex[:location]}) " \ + "– #{pluralize_records(ex[:factories])} created, " \ "#{ex[:time].duration}\n" end msgs << "\n" @@ -87,7 +87,7 @@ def report private def pluralize_records(count) - count == 1 ? "1 record" : "#{count} records" + (count == 1) ? "1 record" : "#{count} records" end end end diff --git a/lib/test_prof/factory_doctor/rspec.rb b/lib/test_prof/factory_doctor/rspec.rb index d4534e35..7c6cecac 100644 --- a/lib/test_prof/factory_doctor/rspec.rb +++ b/lib/test_prof/factory_doctor/rspec.rb @@ -6,6 +6,7 @@ module TestProf module FactoryDoctor class RSpecListener # :nodoc: include Logging + using FloatDuration SUCCESS_MESSAGE = 'FactoryDoctor says: "Looks good to me!"' @@ -67,7 +68,7 @@ def print examples.each do |ex| msgs << " #{ex.description} (#{ex.metadata[:location]}) " \ - "– #{pluralize_records(ex.metadata[:factories])} created, "\ + "– #{pluralize_records(ex.metadata[:factories])} created, " \ "#{ex.metadata[:time].duration}\n" end msgs << "\n" diff --git a/lib/test_prof/factory_prof.rb b/lib/test_prof/factory_prof.rb index 8dad4641..bacca506 100644 --- a/lib/test_prof/factory_prof.rb +++ b/lib/test_prof/factory_prof.rb @@ -3,6 +3,7 @@ require "test_prof/factory_prof/printers/simple" require "test_prof/factory_prof/printers/flamegraph" require "test_prof/factory_prof/printers/nate_heckler" +require "test_prof/factory_prof/printers/json" require "test_prof/factory_prof/factory_builders/factory_bot" require "test_prof/factory_prof/factory_builders/fabrication" @@ -15,25 +16,36 @@ module FactoryProf # FactoryProf configuration class Configuration - attr_accessor :mode, :printer + attr_accessor :mode, :printer, :threshold, :include_variations, :variations_limit, + :truncate_names def initialize - @mode = ENV["FPROF"] == "flamegraph" ? :flamegraph : :simple + @mode = (ENV["FPROF"] == "flamegraph") ? :flamegraph : :simple @printer = case ENV["FPROF"] when "flamegraph" Printers::Flamegraph when "nate_heckler" Printers::NateHeckler + when "json" + Printers::Json else Printers::Simple end + @threshold = ENV.fetch("FPROF_THRESHOLD", 0).to_i + @include_variations = ENV["FPROF_VARS"] == "1" + @variations_limit = ENV.fetch("FPROF_VARIATIONS_LIMIT", 2).to_i + @truncate_names = ENV["FPROF_TRUNCATE_NAMES"] == "1" end # Whether we want to generate flamegraphs def flamegraph? @mode == :flamegraph end + + def include_variations? + @include_variations == true + end end class Result # :nodoc: @@ -46,8 +58,14 @@ def initialize(stacks, raw_stats) # Returns sorted stats def stats - @stats ||= @raw_stats.values - .sort_by { |el| -el[:total_count] } + @stats ||= @raw_stats.values.sort_by { |el| -el[:total_count] }.map do |stat| + unless stat[:variations].empty? + stat = stat.dup + stat[:variations] = stat[:variations].values.sort_by { |nested_el| -nested_el[:total_count] } + end + + stat + end end def total_count @@ -57,14 +75,6 @@ def total_count def total_time @total_time ||= @raw_stats.values.sum { |v| v[:total_time] } end - - private - - def sorted_stats(key) - @raw_stats.values - .map { |el| [el[:name], el[key]] } - .sort_by { |el| -el[1] } - end end class << self @@ -100,15 +110,21 @@ def patch! def run init - printer = config.printer - started_at = TestProf.now - at_exit { printer.dump(result, start_time: started_at) } + at_exit do + print(started_at) + end start end + def print(started_at) + printer = config.printer + + printer.dump(result, start_time: started_at, threshold: config.threshold, truncate_names: config.truncate_names) + end + def start reset! @running = true @@ -122,20 +138,19 @@ def result Result.new(@stacks, @stats) end - def track(factory) + def track(factory, variation:) return yield unless running? @depth += 1 @current_stack << factory if config.flamegraph? - @stats[factory][:total_count] += 1 - @stats[factory][:top_level_count] += 1 if @depth == 1 + track_count(@stats[factory]) + track_count(@stats[factory][:variations][variation_name(variation)]) if config.include_variations? t1 = TestProf.now begin yield ensure t2 = TestProf.now - elapsed = t2 - t1 - @stats[factory][:total_time] += elapsed - @stats[factory][:top_level_time] += elapsed if @depth == 1 + track_time(@stats[factory], t1, t2) + track_time(@stats[factory][:variations][variation_name(variation)], t1, t2) if config.include_variations? @depth -= 1 flush_stack if @depth.zero? end @@ -143,21 +158,46 @@ def track(factory) private + def variation_name(variation) + return "-" if variation.empty? + variations_count = variation.to_s.scan(/\w+/).size + return "[...]" if variations_count > config.variations_limit + + variation + end + def reset! @stacks = [] if config.flamegraph? @depth = 0 @stats = Hash.new do |h, k| - h[k] = { - name: k, - total_count: 0, - top_level_count: 0, - total_time: 0.0, - top_level_time: 0.0 - } + h[k] = hash_template(k) + h[k][:variations] = Hash.new { |hh, variation_key| hh[variation_key] = hash_template(variation_key) } + h[k] end flush_stack end + def hash_template(name) + { + name: name, + total_count: 0, + top_level_count: 0, + total_time: 0.0, + top_level_time: 0.0 + } + end + + def track_count(factory) + factory[:total_count] += 1 + factory[:top_level_count] += 1 if @depth == 1 + end + + def track_time(factory, t1, t2) + elapsed = t2 - t1 + factory[:total_time] += elapsed + factory[:top_level_time] += elapsed if @depth == 1 + end + def flush_stack return unless config.flamegraph? @stacks << @current_stack unless @current_stack.nil? || @current_stack.empty? diff --git a/lib/test_prof/factory_prof/fabrication_patch.rb b/lib/test_prof/factory_prof/fabrication_patch.rb index 542d3542..9c4c7d1b 100644 --- a/lib/test_prof/factory_prof/fabrication_patch.rb +++ b/lib/test_prof/factory_prof/fabrication_patch.rb @@ -5,7 +5,13 @@ module FactoryProf # Wrap #run method with FactoryProf tracking module FabricationPatch def create(name, overrides = {}) - FactoryBuilders::Fabrication.track(name) { super } + variation = "" + + if FactoryProf.config.include_variations? && !overrides.empty? + variation += overrides.keys.sort.to_s.gsub(/[\\":]/, "") + end + + FactoryBuilders::Fabrication.track(name, variation: variation.to_sym) { super } end end end diff --git a/lib/test_prof/factory_prof/factory_bot_patch.rb b/lib/test_prof/factory_prof/factory_bot_patch.rb index 556eb3c7..d2b02414 100644 --- a/lib/test_prof/factory_prof/factory_bot_patch.rb +++ b/lib/test_prof/factory_prof/factory_bot_patch.rb @@ -5,7 +5,21 @@ module FactoryProf # Wrap #run method with FactoryProf tracking module FactoryBotPatch def run(strategy = @strategy) - FactoryBuilders::FactoryBot.track(strategy, @name) { super } + variation = "" + + if FactoryProf.config.include_variations? + if @traits || @overrides + unless @traits.empty? + variation += @traits.sort.join(".").prepend(".") + end + + unless @overrides.empty? + variation += @overrides.keys.sort.to_s.gsub(/[\\":]/, "") + end + end + end + + FactoryBuilders::FactoryBot.track(strategy, @name, variation: variation.to_sym) { super } end end end diff --git a/lib/test_prof/factory_prof/factory_builders/fabrication.rb b/lib/test_prof/factory_prof/factory_builders/fabrication.rb index 688f68b4..5257f76c 100644 --- a/lib/test_prof/factory_prof/factory_builders/fabrication.rb +++ b/lib/test_prof/factory_prof/factory_builders/fabrication.rb @@ -15,8 +15,8 @@ def self.patch end end - def self.track(factory, &block) - FactoryProf.track(factory, &block) + def self.track(factory, **opts, &block) + FactoryProf.track(factory, **opts, &block) end end end diff --git a/lib/test_prof/factory_prof/factory_builders/factory_bot.rb b/lib/test_prof/factory_prof/factory_builders/factory_bot.rb index b43e3bb7..1b9fdc4d 100644 --- a/lib/test_prof/factory_prof/factory_builders/factory_bot.rb +++ b/lib/test_prof/factory_prof/factory_builders/factory_bot.rb @@ -12,15 +12,15 @@ module FactoryBuilders class FactoryBot using TestProf::FactoryBotStrategy - # Monkey-patch FactoryBot / FactoryGirl + # Monkey-patch FactoryBot def self.patch TestProf::FactoryBot::FactoryRunner.prepend(FactoryBotPatch) if defined? TestProf::FactoryBot end - def self.track(strategy, factory, &block) + def self.track(strategy, factory, **opts, &block) return yield unless strategy.create? - FactoryProf.track(factory, &block) + FactoryProf.track(factory, **opts, &block) end end end diff --git a/lib/test_prof/factory_prof/printers/json.rb b/lib/test_prof/factory_prof/printers/json.rb new file mode 100644 index 00000000..4430e079 --- /dev/null +++ b/lib/test_prof/factory_prof/printers/json.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "test_prof/ext/float_duration" + +module TestProf::FactoryProf + module Printers + module Json # :nodoc: all + class << self + using TestProf::FloatDuration + include TestProf::Logging + + def dump(result, start_time:, **) + return log(:info, "No factories detected") if result.raw_stats == {} + + outpath = TestProf.artifact_path("test-prof.result.json") + File.write(outpath, convert_stats(result, start_time).to_json) + + log :info, "Profile results to JSON: #{outpath}" + end + + def convert_stats(result, start_time) + total_run_time = TestProf.now - start_time + total_count = result.stats.sum { |stat| stat[:total_count] } + total_top_level_count = result.stats.sum { |stat| stat[:top_level_count] } + total_time = result.stats.sum { |stat| stat[:top_level_time] } + total_uniq_factories = result.stats.map { |stat| stat[:name] }.uniq.count + + { + total_count: total_count, + total_top_level_count: total_top_level_count, + total_time: total_time.duration, + total_run_time: total_run_time.duration, + total_uniq_factories: total_uniq_factories, + + stats: result.stats + } + end + end + end + end +end diff --git a/lib/test_prof/factory_prof/printers/nate_heckler.rb b/lib/test_prof/factory_prof/printers/nate_heckler.rb index c1ab0d72..bf9d28b6 100644 --- a/lib/test_prof/factory_prof/printers/nate_heckler.rb +++ b/lib/test_prof/factory_prof/printers/nate_heckler.rb @@ -10,7 +10,7 @@ class << self using TestProf::FloatDuration include TestProf::Logging - def dump(result, start_time:) + def dump(result, start_time:, **) return if result.raw_stats == {} total_time = result.stats.sum { |stat| stat[:top_level_time] } diff --git a/lib/test_prof/factory_prof/printers/simple.rb b/lib/test_prof/factory_prof/printers/simple.rb index f85efcd2..307f7ccb 100644 --- a/lib/test_prof/factory_prof/printers/simple.rb +++ b/lib/test_prof/factory_prof/printers/simple.rb @@ -9,7 +9,7 @@ class << self using TestProf::FloatDuration include TestProf::Logging - def dump(result, start_time:) + def dump(result, start_time:, threshold:, truncate_names:) return log(:info, "No factories detected") if result.raw_stats == {} msgs = [] @@ -19,6 +19,12 @@ def dump(result, start_time:) total_time = result.stats.sum { |stat| stat[:top_level_time] } total_uniq_factories = result.stats.map { |stat| stat[:name] }.uniq.count + table_indent = 3 + variations_indent = 2 + max_name_length = result.stats.map { _1[:name].length }.max + max_variation_length = result.stats.flat_map { _1[:variations] }.select(&:present?).map { _1[:name].length }.max || 0 + name_column_length = truncate_names ? 20 : ([max_name_length, max_variation_length].max + variations_indent) + msgs << <<~MSG Factories usage @@ -27,18 +33,61 @@ def dump(result, start_time:) Total top-level: #{total_top_level_count} Total time: #{total_time.duration} (out of #{total_run_time.duration}) Total uniq factories: #{total_uniq_factories} - - total top-level total time time per call top-level time name MSG + msgs << format( + "%#{table_indent}s%-#{name_column_length}s %8s %12s %13s %16s %17s", + "", "name", "total", "top-level", "total time", "time per call", "top-level time" + ) + msgs << "" + result.stats.each do |stat| - time_per_call = stat[:total_time] / stat[:total_count] + next if stat[:total_count] < threshold - msgs << format("%8d %11d %13.4fs %17.4fs %18.4fs %18s", stat[:total_count], stat[:top_level_count], stat[:total_time], time_per_call, stat[:top_level_time], stat[:name]) + msgs << formatted( + table_indent, + name_column_length, + truncate_names, + stat + ) + + # move other variation ("[...]") to the end of the array + sorted_variations = stat[:variations].sort_by.with_index do |variation, i| + (variation[:name] == "[...]") ? stat[:variations].size + 1 : i + end + sorted_variations.each do |variation_stat| + next if variation_stat[:total_count] < threshold + + msgs << formatted( + table_indent + variations_indent, + name_column_length - variations_indent, + truncate_names, + variation_stat + ) + end end log :info, msgs.join("\n") end + + private + + def formatted(indent_len, name_len, truncate_names, stat) + format(format_string(indent_len, name_len, truncate_names), *format_args(stat)) + end + + def format_args(stat) + time_per_call = stat[:total_time] / stat[:total_count] + format_args = [""] + format_args += stat.values_at(:name, :total_count, :top_level_count, :total_time) + format_args << time_per_call + format_args << stat[:top_level_time] + end + + def format_string(indent_len, name_len, truncate_names) + name_format = truncate_names ? "#{name_len}.#{name_len}" : name_len.to_s + "%#{indent_len}s%-#{name_format}s %8d %12d %12.4fs %15.4fs %16.4fs" + end end end end diff --git a/lib/test_prof/logging.rb b/lib/test_prof/logging.rb index 7a16c7f4..6a2aeba0 100644 --- a/lib/test_prof/logging.rb +++ b/lib/test_prof/logging.rb @@ -11,7 +11,7 @@ module Logging class Formatter def call(severity, _time, progname, msg) - colorize(severity.to_sym, "[#{progname} #{severity}] #{msg}\n") + colorize(severity.to_sym, "[#{progname} #{severity}] #{msg}") + "\n" end private diff --git a/lib/test_prof/memory_prof.rb b/lib/test_prof/memory_prof.rb new file mode 100644 index 00000000..a50a1c83 --- /dev/null +++ b/lib/test_prof/memory_prof.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require "test_prof/memory_prof/tracker" +require "test_prof/memory_prof/printer" + +module TestProf + # MemoryProf can help in detecting test examples causing memory spikes. + # It supports two metrics: RSS and allocations. + # + # Example: + # + # TEST_MEM_PROF='rss' rspec ... + # TEST_MEM_PROF='alloc' rspec ... + # + # By default MemoryProf shows the top 5 examples and groups (for RSpec) but you can + # set how many items to display with `TEST_MEM_PROF_COUNT`: + # + # TEST_MEM_PROF='rss' TEST_MEM_PROF_COUNT=10 rspec ... + # + # The examples block shows the amount of memory used by each example, and the groups + # block displays the memory allocated by other code defined in the groups. For example, + # RSpec groups may include heavy `before(:all)` (or `before_all`) setup blocks, so it is + # helpful to see which groups use the most amount of memory outside of their examples. + + module MemoryProf + # MemoryProf configuration + class Configuration + attr_reader :mode, :top_count + + def initialize + self.mode = ENV["TEST_MEM_PROF"] + self.top_count = ENV["TEST_MEM_PROF_COUNT"] + end + + def mode=(value) + @mode = (value == "alloc") ? :alloc : :rss + end + + def top_count=(value) + @top_count = value.to_i + @top_count = 5 unless @top_count.positive? + end + end + + class << self + TRACKERS = { + alloc: AllocTracker, + rss: RssTracker + }.freeze + + PRINTERS = { + alloc: AllocPrinter, + rss: RssPrinter + }.freeze + + def config + @config ||= Configuration.new + end + + def configure + yield config + end + + def tracker + tracker = TRACKERS[config.mode] + tracker.new(config.top_count) + end + + def printer(tracker) + printer = PRINTERS[config.mode] + printer.new(tracker) + end + end + end +end + +require "test_prof/memory_prof/rspec" if TestProf.rspec? +require "test_prof/memory_prof/minitest" if TestProf.minitest? diff --git a/lib/test_prof/memory_prof/minitest.rb b/lib/test_prof/memory_prof/minitest.rb new file mode 100644 index 00000000..077ede68 --- /dev/null +++ b/lib/test_prof/memory_prof/minitest.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require "minitest/base_reporter" + +module Minitest + module TestProf + class MemoryProfReporter < BaseReporter # :nodoc: + attr_reader :tracker, :printer, :current_example + + def initialize(io = $stdout, options = {}) + super + + configure_profiler(options) + + @tracker = ::TestProf::MemoryProf.tracker + @printer = ::TestProf::MemoryProf.printer(tracker) + + @current_example = nil + end + + def prerecord(group, example) + set_current_example(group, example) + tracker.example_started(current_example) + end + + def record(example) + tracker.example_finished(current_example) + end + + def start + tracker.start + end + + def report + tracker.finish + printer.print + end + + private + + def set_current_example(group, example) + @current_example = { + name: example.gsub(/^test_(?:\d+_)?/, ""), + location: location_with_line_number(group, example) + } + end + + def configure_profiler(options) + ::TestProf::MemoryProf.configure do |config| + config.mode = options[:mem_prof_mode] + config.top_count = options[:mem_prof_top_count] if options[:mem_prof_top_count] + end + end + end + end +end diff --git a/lib/test_prof/memory_prof/printer.rb b/lib/test_prof/memory_prof/printer.rb new file mode 100644 index 00000000..7a272d30 --- /dev/null +++ b/lib/test_prof/memory_prof/printer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require "test_prof/memory_prof/printer/number_to_human" +require "test_prof/ext/string_truncate" + +module TestProf + module MemoryProf + class Printer + include Logging + + using StringTruncate + + def initialize(tracker) + @tracker = tracker + end + + def print + messages = [ + "MemoryProf results\n\n", + print_total, + print_block("groups", tracker.groups), + print_block("examples", tracker.examples) + ] + + log :info, messages.join + end + + private + + attr_reader :tracker + + def print_block(name, items) + return if items.empty? + + <<~GROUP + Top #{tracker.top_count} #{name} (by #{mode}): + + #{print_items(items)} + GROUP + end + + def print_items(items) + messages = + items.map do |item| + <<~ITEM + #{item[:name].truncate(30)} (#{item[:location]}) – +#{memory_amount(item)} (#{memory_percentage(item)}%) + ITEM + end + + messages.join + end + + def memory_percentage(item) + return 0 if tracker.total_memory.zero? || item[:memory].zero? + + (100.0 * item[:memory] / tracker.total_memory).round(2) + end + + def number_to_human(value) + NumberToHuman.convert(value) + end + end + + class AllocPrinter < Printer + private + + def mode + "allocations" + end + + def print_total + "Total allocations: #{tracker.total_memory}\n\n" + end + + def memory_amount(item) + item[:memory] + end + end + + class RssPrinter < Printer + private + + def mode + "RSS" + end + + def print_total + "Final RSS: #{number_to_human(tracker.total_memory)}\n\n" + end + + def memory_amount(item) + number_to_human(item[:memory]) + end + end + end +end diff --git a/lib/test_prof/memory_prof/printer/number_to_human.rb b/lib/test_prof/memory_prof/printer/number_to_human.rb new file mode 100644 index 00000000..9c2c3a0f --- /dev/null +++ b/lib/test_prof/memory_prof/printer/number_to_human.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module TestProf + module MemoryProf + class Printer + module NumberToHuman + BASE = 1024 + UNITS = %w[B KB MB GB TB PB EB ZB] + + class << self + def convert(number) + exponent = exponent(number) + human_size = number.to_f / (BASE**exponent) + + "#{round(human_size)}#{UNITS[exponent]}" + end + + private + + def exponent(number) + return 0 unless number.positive? + + max = UNITS.size - 1 + + exponent = (Math.log(number) / Math.log(BASE)).to_i + (exponent > max) ? max : exponent + end + + def round(number) + if integer?(number) + number.round + else + number.round(2) + end + end + + def integer?(number) + number.round == number + end + end + end + end + end +end diff --git a/lib/test_prof/memory_prof/rspec.rb b/lib/test_prof/memory_prof/rspec.rb new file mode 100644 index 00000000..bb6cbcaa --- /dev/null +++ b/lib/test_prof/memory_prof/rspec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module TestProf + module MemoryProf + class RSpecListener + NOTIFICATIONS = %i[ + example_started + example_finished + example_group_started + example_group_finished + ].freeze + + attr_reader :tracker, :printer + + def initialize + @tracker = MemoryProf.tracker + @printer = MemoryProf.printer(tracker) + + @current_group = nil + @current_example = nil + + @tracker.start + end + + def example_started(notification) + tracker.example_started(notification.example, example(notification)) + end + + def example_finished(notification) + tracker.example_finished(notification.example) + end + + def example_group_started(notification) + tracker.group_started(notification.group, group(notification)) + end + + def example_group_finished(notification) + tracker.group_finished(notification.group) + end + + def report + tracker.finish + printer.print + end + + private + + def example(notification) + { + name: notification.example.description, + location: notification.example.metadata[:location] + } + end + + def group(notification) + { + name: notification.group.description, + location: notification.group.metadata[:location] + } + end + end + end +end + +TestProf.activate("TEST_MEM_PROF") do + RSpec.configure do |config| + listener = nil + + config.before(:suite) do + listener = TestProf::MemoryProf::RSpecListener.new + + config.reporter.register_listener( + listener, *TestProf::MemoryProf::RSpecListener::NOTIFICATIONS + ) + end + + config.after(:suite) { listener&.report } + end +end diff --git a/lib/test_prof/memory_prof/tracker.rb b/lib/test_prof/memory_prof/tracker.rb new file mode 100644 index 00000000..c90e9f46 --- /dev/null +++ b/lib/test_prof/memory_prof/tracker.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require "test_prof/memory_prof/tracker/linked_list" +require "test_prof/memory_prof/tracker/rss_tool" + +module TestProf + module MemoryProf + # Tracker is responsible for tracking memory usage and determining + # the top n examples and groups. There are two types of trackers: + # AllocTracker and RssTracker. + # + # A tracker consists of four main parts: + # * list - a linked list that is being used to track memmory for individual groups/examples. + # list is an instance of LinkedList (for more info see tracker/linked_list.rb) + # * examples – the top n examples, an instance of Utils::SizedOrderedSet. + # * groups – the top n groups, an instance of Utils::SizedOrderedSet. + # * track - a method that fetches the amount of memory in use at a certain point. + class Tracker + attr_reader :top_count, :examples, :groups, :total_memory, :list + + def initialize(top_count) + raise "Your Ruby Engine or OS is not supported" unless supported? + + @top_count = top_count + + @examples = Utils::SizedOrderedSet.new(top_count, sort_by: :memory) + @groups = Utils::SizedOrderedSet.new(top_count, sort_by: :memory) + end + + def start + @list = LinkedList.new(track) + end + + def finish + node = list.remove_node(:total, track) + @total_memory = node.total_memory + end + + def example_started(id, example = id) + list.add_node(id, example, track) + end + + def example_finished(id) + node = list.remove_node(id, track) + return unless node + + examples << {**node.item, memory: node.total_memory} + end + + def group_started(id, group = id) + list.add_node(id, group, track) + end + + def group_finished(id) + node = list.remove_node(id, track) + return unless node + + groups << {**node.item, memory: node.hooks_memory} + end + end + + class AllocTracker < Tracker + def track + GC.stat[:total_allocated_objects] + end + + def supported? + RUBY_ENGINE != "jruby" + end + end + + class RssTracker < Tracker + def initialize(top_count) + @rss_tool = RssTool.tool + + super + end + + def track + @rss_tool.track + end + + def supported? + !!@rss_tool + end + end + end +end diff --git a/lib/test_prof/memory_prof/tracker/linked_list.rb b/lib/test_prof/memory_prof/tracker/linked_list.rb new file mode 100644 index 00000000..34a08771 --- /dev/null +++ b/lib/test_prof/memory_prof/tracker/linked_list.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +module TestProf + module MemoryProf + class Tracker + # LinkedList is a linked list that track memory usage for individual examples/groups. + # A list node (`LinkedListNode`) represents an example/group and its memory usage info: + # + # * memory_at_start - the amount of memory at the start of an example/group + # * memory_at_finish - the amount of memory at the end of an example/group + # * nested_memory - the amount of memory allocated by examples/groups defined inside a group + # * previous - a link to the previous node + # + # Each node has a link to its previous node, and the head node points to the current example/group. + # If we picture a linked list as a tree with root being the top level group and leaves being + # current examples/groups, then the head node will always point to a leaf in that tree. + # + # For example, if we have the following spec: + # + # describe Question do + # decribe "#publish" do + # context "when not published" do + # it "makes the question visible" do + # ... + # end + # end + # end + # end + # + # At the moment when rspec is executing the example, the list has the following structure + # (^ denotes the head node): + # + # ^"makes the question visible" -> "when not published" -> "#publish" -> Question + # + # LinkedList supports two method for working with it: + # + # * add_node – adds a node to the beginig of the list. At this point an example or group + # has started and we track how much memory has already been used. + # * remove_node – removes and returns the head node from the list. It means that the node + # example/group has finished and it is time to calculate its memory usage. + # + # When we remove a node we add its total_memory to the previous node.nested_memory, thus + # gradually tracking the amount of memory used by nested examples inside a group. + # + # In the example above, after we remove the node "makes the question visible", we add its total + # memory usage to nested_memory of the "when not published" node. If the "when not published" + # group contains other examples or sub-groups, their total_memory will also be added to + # "when not published" nested_memory. So when the group finishes we will have the total amount + # of memory used by its nested examples/groups, and thus we will be able to calculate the memory + # used by hooks and other code inside a group by subtracting nested_memory from total_memory. + class LinkedList + attr_reader :head + + def initialize(memory_at_start) + add_node(:total, :total, memory_at_start) + end + + def add_node(id, item, memory_at_start) + @head = LinkedListNode.new( + id: id, + item: item, + previous: head, + memory_at_start: memory_at_start + ) + end + + def remove_node(id, memory_at_finish) + return if head.id != id + head.finish(memory_at_finish) + + current = head + @head = head.previous + + current + end + end + + class LinkedListNode + attr_reader :id, :item, :previous, :memory_at_start, :memory_at_finish, :nested_memory + + def initialize(id:, item:, memory_at_start:, previous:) + @id = id + @item = item + @previous = previous + + @memory_at_start = memory_at_start || 0 + @memory_at_finish = nil + @nested_memory = 0 + end + + def total_memory + return 0 if memory_at_finish.nil? + # It seems that on Windows Minitest may release a lot of memory to + # the OS when it finishes and executes #report, leading to memory_at_finish + # being less than memory_at_start. In this case we return nested_memory + # which does not account for the memory used in `after` hooks, but it + # is better than nothing. + return nested_memory if memory_at_start > memory_at_finish + + memory_at_finish - memory_at_start + end + + def hooks_memory + total_memory - nested_memory + end + + def finish(memory_at_finish) + @memory_at_finish = memory_at_finish + + previous&.add_nested(self) + end + + protected + + def add_nested(node) + @nested_memory += node.total_memory + end + end + end + end +end diff --git a/lib/test_prof/memory_prof/tracker/rss_tool.rb b/lib/test_prof/memory_prof/tracker/rss_tool.rb new file mode 100644 index 00000000..b24fe441 --- /dev/null +++ b/lib/test_prof/memory_prof/tracker/rss_tool.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "rbconfig" + +module TestProf + module MemoryProf + class Tracker + module RssTool + class ProcFS + def initialize + @statm = File.open("/proc/#{$$}/statm", "r") + @page_size = get_page_size + end + + def track + @statm.seek(0) + @statm.gets.split(/\s/)[1].to_i * @page_size + end + + private + + def get_page_size + [ + -> { + require "etc" + Etc.sysconf(Etc::SC_PAGE_SIZE) + }, + -> { `getconf PAGE_SIZE`.to_i }, + -> { 0x1000 } + ].each do |strategy| + page_size = begin + strategy.call + rescue + next + end + return page_size + end + end + end + + class PS + def track + `ps -o rss -p #{$$}`.strip.split.last.to_i * 1024 + end + end + + class GetProcess + def track + command.strip.split.last.to_i + end + + private + + def command + `powershell -Command "Get-Process -Id #{$$} | select WS"` + end + end + + TOOLS = { + linux: ProcFS, + macosx: PS, + unix: PS, + windows: GetProcess + }.freeze + + class << self + def tool + TOOLS[os_type]&.new + end + + def os_type + case RbConfig::CONFIG["host_os"] + when /linux/ + :linux + when /darwin|mac os/ + :macosx + when /solaris|bsd/ + :unix + when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ + :windows + end + end + end + end + end + end +end diff --git a/lib/test_prof/recipes/logging.rb b/lib/test_prof/recipes/logging.rb index 60199b90..c815beda 100644 --- a/lib/test_prof/recipes/logging.rb +++ b/lib/test_prof/recipes/logging.rb @@ -1,18 +1,44 @@ # frozen_string_literal: true -require "test_prof" +require "test_prof/core" module TestProf module Rails # Add `with_logging` and `with_ar_logging helpers` module LoggingHelpers class << self - attr_writer :logger + def logger=(logger) + @logger = logger + + # swap global loggers + global_loggables.each do |loggable| + loggable.logger = logger + end + end def logger return @logger if instance_variable_defined?(:@logger) - @logger = Logger.new($stdout) + @logger = if defined?(ActiveSupport::TaggedLogging) + ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout)) + elsif defined?(ActiveSupport::Logger) + ActiveSupport::Logger.new($stdout) + else + Logger.new($stdout) + end + end + + def global_loggables + return @global_loggables if instance_variable_defined?(:@global_loggables) + + @global_loggables = [] + end + + def swap_logger!(loggables) + loggables.each do |loggable| + loggable.logger = logger + global_loggables << loggable + end end def ar_loggables @@ -24,8 +50,6 @@ def ar_loggables ] end - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity def all_loggables return @all_loggables if instance_variable_defined?(:@all_loggables) @@ -60,16 +84,30 @@ def restore_logger(was_loggers, loggables) # Enable verbose Rails logging within a block def with_logging + if ::ActiveSupport.respond_to?(:event_reporter) + @was_events_debug_mode = ActiveSupport.event_reporter.debug_mode? + ::ActiveSupport.event_reporter.debug_mode = true + end *loggers = LoggingHelpers.swap_logger(LoggingHelpers.all_loggables) yield ensure + if ::ActiveSupport.respond_to?(:event_reporter) + ::ActiveSupport.event_reporter.debug_mode = @was_events_debug_mode + end LoggingHelpers.restore_logger(loggers, LoggingHelpers.all_loggables) end def with_ar_logging + if ::ActiveSupport.respond_to?(:event_reporter) + @was_events_debug_mode = ActiveSupport.event_reporter.debug_mode? + ::ActiveSupport.event_reporter.debug_mode = true + end *loggers = LoggingHelpers.swap_logger(LoggingHelpers.ar_loggables) yield ensure + if ::ActiveSupport.respond_to?(:event_reporter) + ::ActiveSupport.event_reporter.debug_mode = @was_events_debug_mode + end LoggingHelpers.restore_logger(loggers, LoggingHelpers.ar_loggables) end end @@ -79,7 +117,9 @@ def with_ar_logging if TestProf.rspec? RSpec.shared_context "logging:verbose" do around(:each) do |ex| - with_logging(&ex) + next with_logging(&ex) if ex.metadata[:log] == true || ex.metadata[:log] == :all + + ex.call end end @@ -98,13 +138,16 @@ def with_ar_logging TestProf.activate("LOG", "all") do TestProf.log :info, "Rails verbose logging enabled" - ActiveSupport::LogSubscriber.logger = - Rails.logger = - ActiveRecord::Base.logger = TestProf::Rails::LoggingHelpers.logger + if ::ActiveSupport.respond_to?(:event_reporter) + ::ActiveSupport.event_reporter.debug_mode = true + end + TestProf::Rails::LoggingHelpers.swap_logger!(TestProf::Rails::LoggingHelpers.all_loggables) end TestProf.activate("LOG", "ar") do TestProf.log :info, "Active Record verbose logging enabled" - ActiveSupport::LogSubscriber.logger = - ActiveRecord::Base.logger = TestProf::Rails::LoggingHelpers.logger + if ::ActiveSupport.respond_to?(:event_reporter) + ::ActiveSupport.event_reporter.debug_mode = true + end + TestProf::Rails::LoggingHelpers.swap_logger!(TestProf::Rails::LoggingHelpers.ar_loggables) end diff --git a/lib/test_prof/recipes/minitest/before_all.rb b/lib/test_prof/recipes/minitest/before_all.rb index ef5337b5..da43b85b 100644 --- a/lib/test_prof/recipes/minitest/before_all.rb +++ b/lib/test_prof/recipes/minitest/before_all.rb @@ -2,6 +2,22 @@ require "test_prof/before_all" +Minitest.singleton_class.prepend(Module.new do + attr_reader :previous_klass + @previous_klass = nil + + def run_one_method(klass, method_name) + return super unless klass.respond_to?(:parallelized) && klass.parallelized + + if @previous_klass && @previous_klass != klass + @previous_klass.before_all_executor&.deactivate! + end + @previous_klass = klass + + super + end +end) + module TestProf module BeforeAll # Add before_all hook to Minitest: wrap all examples into a transaction and @@ -83,6 +99,38 @@ def perform_teardown(test_object) class << self def included(base) base.extend ClassMethods + + base.cattr_accessor :parallelized + if base.respond_to?(:parallelize_teardown) + base.parallelize_teardown do + last_klass = ::Minitest.previous_klass + if last_klass&.respond_to?(:parallelized) && last_klass.parallelized + last_klass.before_all_executor&.deactivate! + end + end + end + + if base.respond_to?(:parallelize) + base.singleton_class.prepend(Module.new do + def parallelize(workers: :number_of_processors, with: :processes) + # super.parallelize returns nil when no parallelization is set up + if super.nil? + return + end + + case with + when :processes + self.parallelized = true + when :threads + warn "!!! before_all is not implemented for parallalization with threads and " \ + "could work incorrectly" + else + warn "!!! tests are using an unknown parallelization strategy and before_all " \ + "could work incorrectly" + end + end + end) + end end end @@ -118,7 +166,7 @@ def before_setup def run(*) super ensure - before_all_executor&.deactivate! + before_all_executor&.deactivate! unless parallelized end end) end diff --git a/lib/test_prof/recipes/minitest/sample.rb b/lib/test_prof/recipes/minitest/sample.rb index 69976906..ba6a1810 100644 --- a/lib/test_prof/recipes/minitest/sample.rb +++ b/lib/test_prof/recipes/minitest/sample.rb @@ -6,15 +6,17 @@ module MinitestSample # Do not add these classes to resulted sample CORE_RUNNABLES = [ Minitest::Test, - Minitest::Unit::TestCase, - Minitest::Spec - ].freeze + defined?(Minitest::Unit::TestCase) ? Minitest::Unit::TestCase : nil, + defined?(Minitest::Spec) ? Minitest::Spec : nil + ].compact.freeze class << self def suites # Make sure that sample contains only _real_ suites Minitest::Runnable.runnables - .reject { |suite| CORE_RUNNABLES.include?(suite) } + .select do |suite| + CORE_RUNNABLES.any? { |kl| suite < kl } && suite.runnable_methods.any? + end end def sample_groups(sample_size) @@ -27,11 +29,18 @@ def sample_examples(sample_size) all_examples = suites.flat_map do |runnable| runnable.runnable_methods.map { |method| [runnable, method] } end - sample = all_examples.sample(sample_size) + + sample = all_examples.sample(sample_size).group_by(&:first) + sample.transform_values! { |v| v.map(&:last) } + # Filter examples by overriding #runnable_methods for all suites suites.each do |runnable| - runnable.define_singleton_method(:runnable_methods) do - super() & sample.select { |ex| ex.first.equal?(runnable) }.map(&:last) + if sample.key?(runnable) + runnable.define_singleton_method(:runnable_methods) do + super() & sample[runnable] + end + else + runnable.define_singleton_method(:runnable_methods) { [] } end end end diff --git a/lib/test_prof/recipes/rspec/any_fixture.rb b/lib/test_prof/recipes/rspec/any_fixture.rb index 307e3dfe..a97e6172 100644 --- a/lib/test_prof/recipes/rspec/any_fixture.rb +++ b/lib/test_prof/recipes/rspec/any_fixture.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true require "test_prof/any_fixture" +require "test_prof/any_fixture/dsl" +require "test_prof/ext/active_record_refind" + require "test_prof/recipes/rspec/before_all" RSpec.shared_context "any_fixture:clean" do @@ -8,6 +11,11 @@ before_all do TestProf::AnyFixture.clean + TestProf::AnyFixture.disable! + end + + after(:all) do + TestProf::AnyFixture.enable! end end diff --git a/lib/test_prof/recipes/rspec/before_all.rb b/lib/test_prof/recipes/rspec/before_all.rb index 44060939..85d664bb 100644 --- a/lib/test_prof/recipes/rspec/before_all.rb +++ b/lib/test_prof/recipes/rspec/before_all.rb @@ -12,28 +12,31 @@ def before_all(setup_fixtures: BeforeAll.config.setup_fixtures, &block) if within_before_all? before(:all) do @__inspect_output = "before_all hook" + ::RSpec.current_scope = :before_all if ::RSpec.respond_to?(:current_scope=) instance_eval(&block) end return end - @__before_all_activated__ = true + @__before_all_activation__ = context = self + current_metadata = metadata before(:all) do @__inspect_output = "before_all hook" + ::RSpec.current_scope = :before_all if ::RSpec.respond_to?(:current_scope=) BeforeAll.setup_fixtures(self) if setup_fixtures - BeforeAll.begin_transaction do + BeforeAll.begin_transaction(context, current_metadata) do instance_eval(&block) end end after(:all) do - BeforeAll.rollback_transaction + BeforeAll.rollback_transaction(context, current_metadata) end end def within_before_all? - instance_variable_defined?(:@__before_all_activated__) + instance_variable_defined?(:@__before_all_activation__) end end end diff --git a/lib/test_prof/recipes/rspec/factory_default.rb b/lib/test_prof/recipes/rspec/factory_default.rb index c56f85cd..ca4601ae 100644 --- a/lib/test_prof/recipes/rspec/factory_default.rb +++ b/lib/test_prof/recipes/rspec/factory_default.rb @@ -4,6 +4,25 @@ TestProf::FactoryDefault.init +if defined?(TestProf::BeforeAll) + TestProf::BeforeAll.configure do |config| + config.before(:begin) do |context| + TestProf::FactoryDefault.current_context = context + end + + config.after(:rollback) do |context| + TestProf::FactoryDefault.reset(context: context) + end + end +end + RSpec.configure do |config| - config.after(:each) { TestProf::FactoryDefault.reset } + if defined?(TestProf::BeforeAll) + config.before(:each) { TestProf::FactoryDefault.current_context = :example } + config.after(:each) { TestProf::FactoryDefault.reset(context: :example) } + else + config.after(:each) { TestProf::FactoryDefault.reset } + end + + config.after(:suite) { TestProf::FactoryDefault.print_report } end diff --git a/lib/test_prof/recipes/rspec/let_it_be.rb b/lib/test_prof/recipes/rspec/let_it_be.rb index 84feee7d..b2406d9f 100644 --- a/lib/test_prof/recipes/rspec/let_it_be.rb +++ b/lib/test_prof/recipes/rspec/let_it_be.rb @@ -1,13 +1,23 @@ # frozen_string_literal: true -require "test_prof" -require_relative "./before_all" +require "test_prof/core" +require "test_prof/recipes/rspec/before_all" module TestProf # Just like `let`, but persist the result for the whole group. # NOTE: Experimental and magical, for more control use `before_all`. module LetItBe + class DuplicationError < StandardError; end + + Modifier = Struct.new(:scope, :block) do + def call(record, config) + block.call(record, config) + end + end + class Configuration + attr_accessor :report_duplicates + # Define an alias for `let_it_be` with the predefined options: # # TestProf::LetItBe.configure do |config| @@ -17,10 +27,13 @@ def alias_to(name, **default_args) LetItBe.define_let_it_be_alias(name, **default_args) end - def register_modifier(key, &block) + # Register modifier by providing the name of key, + # optional scope (when to apply the modifier, on initialization (:initialize) + # or when accessed # via let (:let)) + def register_modifier(key, on: :let, &block) raise ArgumentError, "Modifier #{key} is already defined for let_it_be" if LetItBe.modifiers.key?(key) - LetItBe.modifiers[key] = block + LetItBe.modifiers[key] = Modifier.new(on, block) end def default_modifiers @@ -41,17 +54,18 @@ def modifiers @modifiers ||= {} end - def wrap_with_modifiers(mods, &block) + def wrap_with_modifiers(mods, on: :let, &block) + mods = mods.select { |k, val| LetItBe.modifiers.fetch(k).scope == on } return block if mods.empty? validate_modifiers! mods - -> { + proc do record = instance_eval(&block) mods.inject(record) do |rec, (k, v)| LetItBe.modifiers.fetch(k).call(rec, v) end - } + end end def module_for(group) @@ -72,10 +86,12 @@ def validate_modifiers!(mods) "Available modifiers are: #{modifiers.keys.join(", ")}" end end + # Use uniq prefix for instance variables to avoid collisions # We want to use the power of Ruby's unicode support) # And we love cats!) - PREFIX = RUBY_ENGINE == "jruby" ? "@__jruby_is_not_cat_friendly__" : "@😸" + # Allow overriding the prefix (there are some intermittent issues on JRuby still) + PREFIX = ENV.fetch("LET_IT_BE_IVAR_PREFIX", "@😸") FROZEN_ERROR_HINT = "\nIf you are using `let_it_be`, you may want to pass `reload: true` or `refind: true` modifier to it." @@ -97,20 +113,26 @@ def let_it_be(identifier, **options, &block) options = default_options.merge(options) + initializer = LetItBe.wrap_with_modifiers(options, on: :initialize, &initializer) + before_all(&initializer) - let_accessor = LetItBe.wrap_with_modifiers(options) do + let_accessor = LetItBe.wrap_with_modifiers(options, on: :let) do instance_variable_get(:"#{PREFIX}#{identifier}") end + report_duplicates(identifier) if LetItBe.config.report_duplicates + LetItBe.module_for(self).module_eval do define_method(identifier) do # Trying to detect the context - # Based on https://github.com/rspec/rspec-rails/commit/7cb796db064f58da7790a92e73ab906ef50b1f34 - if @__inspect_output.include?("before(:context)") || @__inspect_output.include?("before_all") + # First, check for ::RSpec.current_scope (modern RSpec) and then read @__inspect_output + # (based on https://github.com/rspec/rspec-rails/commit/7cb796db064f58da7790a92e73ab906ef50b1f34) + if ::RSpec.respond_to?(:current_scope) && %i[before_all before_context_hook after_context_hook].include?(::RSpec.current_scope) + instance_variable_get(:"#{PREFIX}#{identifier}") + elsif /(before|after)\(:context\)/.match?(@__inspect_output) || @__inspect_output.include?("before_all") instance_variable_get(:"#{PREFIX}#{identifier}") else - # Fallback to let definition super() end end @@ -119,6 +141,19 @@ def let_it_be(identifier, **options, &block) let(identifier, &let_accessor) end + private def report_duplicates(identifier) + if method_defined?(identifier) && File.basename(__FILE__) == File.basename(instance_method(identifier).source_location[0]) + error_msg = "let_it_be(:#{identifier}) was redefined in nested group" + report_level = LetItBe.config.report_duplicates.to_sym + + if report_level == :warn + ::RSpec.warn_with(error_msg) + elsif report_level == :raise + raise DuplicationError, error_msg + end + end + end + module Freezer # Stoplist to prevent freezing objects and theirs associations that are defined # with `let_it_be`'s `freeze: false` options during deep freezing. @@ -192,7 +227,7 @@ def deep_freeze(record) next record.reload if record.is_a?(::ActiveRecord::Base) - if record.respond_to?(:map) + if record.respond_to?(:to_ary) next record.map do |rec| rec.is_a?(::ActiveRecord::Base) ? rec.reload : rec end @@ -205,7 +240,7 @@ def deep_freeze(record) next record.refind if record.is_a?(::ActiveRecord::Base) - if record.respond_to?(:map) + if record.respond_to?(:to_ary) next record.map do |rec| rec.is_a?(::ActiveRecord::Base) ? rec.refind : rec end @@ -213,7 +248,7 @@ def deep_freeze(record) record end - config.register_modifier :freeze do |record, val| + config.register_modifier :freeze, on: :initialize do |record, val| if val == false TestProf::LetItBe::Freezer::Stoplist.stop!(record) next record diff --git a/lib/test_prof/rspec_dissect.rb b/lib/test_prof/rspec_dissect.rb index 2a9769e4..0ddfc108 100644 --- a/lib/test_prof/rspec_dissect.rb +++ b/lib/test_prof/rspec_dissect.rb @@ -47,7 +47,7 @@ def initialize @let_top_count = (ENV["RD_PROF_LET_TOP"] || 3).to_i @top_count = (ENV["RD_PROF_TOP"] || 5).to_i @stamp = ENV["RD_PROF_STAMP"] - @mode = ENV["RD_PROF"] == "1" ? "all" : ENV["RD_PROF"] + @mode = (ENV["RD_PROF"] == "1") ? "all" : ENV["RD_PROF"] unless MODES.include?(mode) raise "Unknown RSpecDissect mode: #{mode};" \ diff --git a/lib/test_prof/rspec_dissect/collectors/before.rb b/lib/test_prof/rspec_dissect/collectors/before.rb index 64fb1a77..ea5608e8 100644 --- a/lib/test_prof/rspec_dissect/collectors/before.rb +++ b/lib/test_prof/rspec_dissect/collectors/before.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "./base" +require "test_prof/rspec_dissect/collectors/base" module TestProf module RSpecDissect diff --git a/lib/test_prof/rspec_dissect/collectors/let.rb b/lib/test_prof/rspec_dissect/collectors/let.rb index ae858fbc..d7f885dc 100644 --- a/lib/test_prof/rspec_dissect/collectors/let.rb +++ b/lib/test_prof/rspec_dissect/collectors/let.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "./base" +require "test_prof/rspec_dissect/collectors/base" module TestProf module RSpecDissect @@ -29,7 +29,7 @@ def print_group_result(group) .sort_by(&:last) .take(RSpecDissect.config.let_top_count) .each do |(id, size)| - msgs << " ↳ #{id} – #{-size}\n" + msgs << " ↳ #{id} – #{-size}\n" end msgs.join end diff --git a/lib/test_prof/rspec_dissect/rspec.rb b/lib/test_prof/rspec_dissect/rspec.rb index 84da5865..6b7ae2e4 100644 --- a/lib/test_prof/rspec_dissect/rspec.rb +++ b/lib/test_prof/rspec_dissect/rspec.rb @@ -6,6 +6,7 @@ module TestProf module RSpecDissect class Listener # :nodoc: include Logging + using FloatDuration NOTIFICATIONS = %i[ diff --git a/lib/test_prof/rspec_stamp.rb b/lib/test_prof/rspec_stamp.rb index c4d5b4f2..841516f7 100644 --- a/lib/test_prof/rspec_stamp.rb +++ b/lib/test_prof/rspec_stamp.rb @@ -116,8 +116,6 @@ def apply_tags(code, lines, tags) private - # rubocop: disable Metrics/CyclomaticComplexity - # rubocop: disable Metrics/PerceivedComplexity def stamp_example(example, tags) matches = example.match(EXAMPLE_RXP) return false unless matches @@ -155,14 +153,16 @@ def stamp_example(example, tags) end end - replacement = "\\1#{parsed.fname}#{need_parens ? "(" : " "}"\ - "#{[desc, tags_str, htags_str].compact.join(", ")}"\ - "#{need_parens ? ") " : " "}\\3" + replacement = proc do + $1 + "#{parsed.fname}#{need_parens ? "(" : " "}" \ + "#{[desc, tags_str, htags_str].compact.join(", ")}" \ + "#{need_parens ? ") " : " "}" + $3 + end if config.dry_run? - log :info, "Patched: #{example.sub(EXAMPLE_RXP, replacement)}" + log :info, "Patched: #{example.sub(EXAMPLE_RXP, &replacement)}" else - example.sub!(EXAMPLE_RXP, replacement) + example.sub!(EXAMPLE_RXP, &replacement) end true end diff --git a/lib/test_prof/rspec_stamp/parser.rb b/lib/test_prof/rspec_stamp/parser.rb index 3f3000f5..3a1d01b8 100644 --- a/lib/test_prof/rspec_stamp/parser.rb +++ b/lib/test_prof/rspec_stamp/parser.rb @@ -28,8 +28,6 @@ def remove_tag(tag) end class << self - # rubocop: disable Metrics/CyclomaticComplexity - # rubocop: disable Metrics/PerceivedComplexity def parse(code) sexp = Ripper.sexp(code) return unless sexp @@ -145,7 +143,7 @@ def parse_const(expr) elsif expr.first == :@const expr[1] elsif expr.first == :const_path_ref - expr[1..-1].map(&method(:parse_const)).join("::") + expr[1..].map(&method(:parse_const)).join("::") end end end diff --git a/lib/test_prof/rubocop.rb b/lib/test_prof/rubocop.rb index c7ebdada..c9a361bb 100644 --- a/lib/test_prof/rubocop.rb +++ b/lib/test_prof/rubocop.rb @@ -1,13 +1,14 @@ # frozen_string_literal: true -require "test_prof/utils" -supported = TestProf::Utils.verify_gem_version("rubocop", at_least: "0.51.0") -unless supported - warn "TestProf cops require RuboCop >= 0.51.0 to run." - return -end +warn <<~MSG + !!! -require "rubocop" + Please, update your .rubocop.yml configuration to load TestProf plugin as follows (and fix the error below): -require_relative "cops/inject" -require "test_prof/cops/rspec/aggregate_examples" + plugins: + - test-prof + + !!! + + +MSG diff --git a/lib/test_prof/ruby_prof.rb b/lib/test_prof/ruby_prof.rb index b9867b0b..a99b4f18 100644 --- a/lib/test_prof/ruby_prof.rb +++ b/lib/test_prof/ruby_prof.rb @@ -48,12 +48,13 @@ class Configuration attr_accessor :printer, :mode, :min_percent, :include_threads, :exclude_common_methods, :test_prof_exclusions_enabled, - :custom_exclusions + :custom_exclusions, :skip_boot def initialize @printer = ENV["TEST_RUBY_PROF"].to_sym if PRINTERS.key?(ENV["TEST_RUBY_PROF"]) @printer ||= ENV.fetch("TEST_RUBY_PROF_PRINTER", :flat).to_sym - @mode = ENV.fetch("TEST_RUBY_PROF_MODE", :wall).to_sym + @mode = ENV.fetch("TEST_RUBY_PROF_MODE", :wall).to_s + @skip_boot = %w[0 false f].include?(ENV["TEST_RUBY_PROF_BOOT"]) @min_percent = 1 @include_threads = false @exclude_common_methods = true @@ -65,6 +66,10 @@ def include_threads? include_threads == true end + def skip_boot? + skip_boot == true + end + def exclude_common_methods? exclude_common_methods == true end @@ -84,6 +89,22 @@ def resolve_printer [type, ::RubyProf.const_get(PRINTERS[type])] end + + # Based on deprecated https://github.com/ruby-prof/ruby-prof/blob/fd3a5236a459586c5ca7ce4de506c1835129516a/lib/ruby-prof.rb#L36 + def ruby_prof_mode + case mode + when "wall", "wall_time" + ::RubyProf::WALL_TIME + when "allocations" + ::RubyProf::ALLOCATIONS + when "memory" + ::RubyProf::MEMORY + when "process", "process_time" + ::RubyProf::PROCESS_TIME + else + ::RubyProf::WALL_TIME + end + end end # Wrapper over RubyProf profiler and printer @@ -150,35 +171,32 @@ def configure # # Use this method to profile the whole run. def run - report = profile + report = profile(locked: true) return unless report - @locked = true - log :info, "RubyProf enabled globally" at_exit { report.dump("total") } end - def profile + def profile(locked: false) if locked? log :warn, <<~MSG RubyProf is activated globally, you cannot generate per-example report. - Make sure you haven's set the TEST_RUBY_PROF environmental variable. + Make sure you haven not set the TEST_RUBY_PROF environmental variable. MSG return end return unless init_ruby_prof - options = { - merge_fibers: true - } + options = {} options[:include_threads] = [Thread.current] unless config.include_threads? + options[:measure_mode] = config.ruby_prof_mode profiler = ::RubyProf::Profile.new(options) profiler.exclude_common_methods! if config.exclude_common_methods? @@ -197,6 +215,8 @@ def profile profiler.start + @locked = true if locked + Report.new(profiler) end @@ -208,13 +228,12 @@ def locked? def init_ruby_prof return @initialized if instance_variable_defined?(:@initialized) - ENV["RUBY_PROF_MEASURE_MODE"] = config.mode.to_s @initialized = TestProf.require( "ruby-prof", <<~MSG Please, install 'ruby-prof' first: # Gemfile - gem 'ruby-prof', '>= 0.16.0', require: false + gem 'ruby-prof', '>= 1.4.0', require: false MSG ) { check_ruby_prof_version } end @@ -268,5 +287,13 @@ def exclude_common_methods(profiler) # Hook to run RubyProf globally TestProf.activate("TEST_RUBY_PROF") do - TestProf::RubyProf.run + if TestProf::RubyProf.config.skip_boot? + if TestProf.rspec? + require "test_prof/ruby_prof/rspec_no_boot" + else + TestProf.log :warn, "RubyProf tests profiling w/o test suite boot is only supported in RSpec" + end + else + TestProf::RubyProf.run + end end diff --git a/lib/test_prof/ruby_prof/rspec_no_boot.rb b/lib/test_prof/ruby_prof/rspec_no_boot.rb new file mode 100644 index 00000000..0066e01a --- /dev/null +++ b/lib/test_prof/ruby_prof/rspec_no_boot.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + report = nil + + config.append_before(:suite) do + report = TestProf::RubyProf.profile(locked: true) + + TestProf.log :info, "RubyProf enabled for examples" + end + + config.after(:suite) do + report&.dump("examples") + end +end diff --git a/lib/test_prof/stack_prof.rb b/lib/test_prof/stack_prof.rb index bd4c9324..5f99a988 100644 --- a/lib/test_prof/stack_prof.rb +++ b/lib/test_prof/stack_prof.rb @@ -24,21 +24,21 @@ module StackProf class Configuration FORMATS = %w[html json].freeze - attr_accessor :mode, :interval, :raw, :target, :format - + attr_accessor :mode, :raw, :target, :format, :interval, :ignore_gc def initialize @mode = ENV.fetch("TEST_STACK_PROF_MODE", :wall).to_sym - @target = ENV["TEST_STACK_PROF"] == "boot" ? :boot : :suite + @target = (ENV["TEST_STACK_PROF"] == "boot") ? :boot : :suite @raw = ENV["TEST_STACK_PROF_RAW"] != "0" @format = if FORMATS.include?(ENV["TEST_STACK_PROF_FORMAT"]) ENV["TEST_STACK_PROF_FORMAT"] else - "html" + "json" end sample_interval = ENV["TEST_STACK_PROF_INTERVAL"].to_i - @interval = sample_interval > 0 ? sample_interval : nil + @interval = (sample_interval > 0) ? sample_interval : nil + @ignore_gc = !ENV["TEST_STACK_PROF_IGNORE_GC"].nil? end def raw? @@ -72,7 +72,7 @@ def run @locked = true - log :info, "StackProf#{config.raw? ? " (raw)" : ""} enabled globally: " \ + log :info, "StackProf#{" (raw)" if config.raw?} enabled globally: " \ "mode – #{config.mode}, target – #{config.target}" at_exit { dump("total") } if config.suite? @@ -81,9 +81,9 @@ def run def profile(name = nil) if locked? log :warn, <<~MSG - StackProf is activated globally, you cannot generate per-example report. + StackProf has been already activated. - Make sure you haven's set the TEST_STACK_PROF environmental variable. + Make sure you have not set the TEST_STACK_PROF environmental variable. MSG return false end @@ -96,6 +96,7 @@ def profile(name = nil) } options[:interval] = config.interval if config.interval + options[:ignore_gc] = true if config.ignore_gc if block_given? options[:out] = build_path(name) @@ -124,7 +125,7 @@ def dump(name) def build_path(name) TestProf.artifact_path( - "stack-prof-report-#{config.mode}#{config.raw ? "-raw" : ""}-#{name}.dump" + "stack-prof-report-#{config.mode}#{"-raw" if config.raw}-#{name}.dump" ) end @@ -162,13 +163,13 @@ def dump_html_report(path) log :info, <<~MSG Run the following command to generate a flame graph report: - stackprof --flamegraph #{path} > #{html_path} && stackprof --flamegraph-viewer=#{html_path} + stackprof --d3-flamegraph #{path} > #{html_path} && stackprof --flamegraph-viewer=#{html_path} MSG end def dump_json_report(path) report = ::StackProf::Report.new( - Marshal.load(IO.binread(path)) # rubocop:disable Security/MarshalLoad + Marshal.load(IO.binread(path)) ) json_path = path.gsub(/\.dump$/, ".json") File.write(json_path, JSON.generate(report.data)) diff --git a/lib/test_prof/stack_prof/rspec.rb b/lib/test_prof/stack_prof/rspec.rb index 3457daad..cf7e9642 100644 --- a/lib/test_prof/stack_prof/rspec.rb +++ b/lib/test_prof/stack_prof/rspec.rb @@ -19,12 +19,13 @@ class << self def example_started(notification) return unless profile?(notification.example) + notification.example.metadata[:sprof_report] = TestProf::StackProf.profile end def example_finished(notification) return unless profile?(notification.example) - return unless notification.example.metadata[:sprof_report] == false + return if notification.example.metadata[:sprof_report] == false TestProf::StackProf.dump( self.class.report_name_generator.call(notification.example) diff --git a/lib/test_prof/tag_prof/minitest.rb b/lib/test_prof/tag_prof/minitest.rb new file mode 100644 index 00000000..6b2b6f46 --- /dev/null +++ b/lib/test_prof/tag_prof/minitest.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Minitest + module TestProf + class TagProfReporter < BaseReporter # :nodoc: + attr_reader :results + + def initialize(io = $stdout, _options = {}) + super + @results = ::TestProf::TagProf::Result.new("type") + + if event_prof_activated? + require "test_prof/event_prof" + @current_group_id = nil + @events_profiler = configure_profiler + @results = ::TestProf::TagProf::Result.new("type", @events_profiler.events) + end + end + + def prerecord(group, example) + return unless event_prof_activated? + + # enable event profiling + @events_profiler.group_started(true) + end + + def record(result) + results.track(main_folder_path(result), time: result.time, events: fetch_events_data) + @events_profiler.group_started(nil) if event_prof_activated? # reset and disable event profilers + end + + def report + printer = (ENV["TAG_PROF_FORMAT"] == "html") ? ::TestProf::TagProf::Printers::HTML : ::TestProf::TagProf::Printers::Simple + printer.dump(results) + end + + private + + def main_folder_path(result) + return :__unknown__ if absolute_path_from(result).nil? + + absolute_path_from(result) + end + + def absolute_path_from(result) + absolute_path = File.expand_path(result.source_location.first) + absolute_path.slice(/(?<=(?:spec|test)\/)\w*/) + end + + def configure_profiler + ::TestProf::EventProf::CustomEvents.activate_all(tag_prof_event) + ::TestProf::EventProf.build(tag_prof_event) + end + + def event_prof_activated? + return false if tag_prof_event.nil? + + !tag_prof_event.empty? + end + + def tag_prof_event + ENV["TAG_PROF_EVENT"] + end + + def fetch_events_data + return {} unless @events_profiler + + @events_profiler.profilers.map do |profiler| + [profiler.event, profiler.time || 0.0] + end.to_h + end + end + end +end diff --git a/lib/test_prof/tag_prof/printers/simple.rb b/lib/test_prof/tag_prof/printers/simple.rb index 632bae4d..03ab15d4 100644 --- a/lib/test_prof/tag_prof/printers/simple.rb +++ b/lib/test_prof/tag_prof/printers/simple.rb @@ -7,6 +7,7 @@ module Printers module Simple # :nodoc: all class << self include TestProf::Logging + using TestProf::FloatDuration def dump(result) @@ -35,7 +36,7 @@ def dump(result) ) end - header << format( + header << format( # rubocop:disable Style/RedundantFormat "%6s %6s %6s %12s", "total", "%total", "%time", "avg" ) diff --git a/lib/test_prof/tag_prof/result.rb b/lib/test_prof/tag_prof/result.rb index 73c07b08..fa3f708d 100644 --- a/lib/test_prof/tag_prof/result.rb +++ b/lib/test_prof/tag_prof/result.rb @@ -21,8 +21,8 @@ def initialize(tag, events = []) def track(tag, time:, events: {}) data[tag][:count] += 1 data[tag][:time] += time - events.each do |k, v| - data[tag][k] += v + events.each do |event, time| + data[tag][event] += time end end diff --git a/lib/test_prof/tag_prof/rspec.rb b/lib/test_prof/tag_prof/rspec.rb index a974b360..734e402a 100644 --- a/lib/test_prof/tag_prof/rspec.rb +++ b/lib/test_prof/tag_prof/rspec.rb @@ -13,7 +13,7 @@ class RSpecListener # :nodoc: attr_reader :result, :printer def initialize - @printer = ENV["TAG_PROF_FORMAT"] == "html" ? Printers::HTML : Printers::Simple + @printer = (ENV["TAG_PROF_FORMAT"] == "html") ? Printers::HTML : Printers::Simple @result = if ENV["TAG_PROF_EVENT"].nil? diff --git a/lib/test_prof/tps_prof.rb b/lib/test_prof/tps_prof.rb new file mode 100644 index 00000000..79c733f4 --- /dev/null +++ b/lib/test_prof/tps_prof.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test_prof/tps_prof/profiler" +require "test_prof/tps_prof/reporter/text" + +module TestProf + # TPSProf shows top-N example group based on their tests-per-second value. + # + # Example: + # + # TPS_PROF=10 rspec ... + # + module TPSProf + class Configuration + attr_accessor :top_count, :threshold, :reporter + + def initialize + @top_count = ENV["TPS_PROF"].to_i + @top_count = 10 if @top_count == 1 + @threshold = ENV.fetch("TPS_PROF_MIN", 10).to_i + @reporter = resolve_reporter(ENV["TPS_PROF_FORMAT"]) + end + + private + + def resolve_reporter(format) + # TODO: support other formats + TPSProf::Reporter::Text.new + end + end + + class << self + def config + @config ||= Configuration.new + end + + def configure + yield config + end + end + end +end + +require "test_prof/tps_prof/rspec" if TestProf.rspec? +# TODO: Minitest support +# require "test_prof/tps_prof/minitest" if TestProf.minitest? diff --git a/lib/test_prof/tps_prof/profiler.rb b/lib/test_prof/tps_prof/profiler.rb new file mode 100644 index 00000000..11f720ec --- /dev/null +++ b/lib/test_prof/tps_prof/profiler.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require "test_prof/utils/sized_ordered_set" + +module TestProf + module TPSProf + class Profiler + attr_reader :top_count, :groups, :total_count, :total_time, :threshold + + def initialize(top_count, threshold: 10) + @threshold = threshold + @top_count = top_count + @total_count = 0 + @total_time = 0.0 + @groups = Utils::SizedOrderedSet.new(top_count, sort_by: :tps) + end + + def group_started(id) + @current_group = id + @examples_count = 0 + @examples_time = 0.0 + @group_started_at = TestProf.now + end + + def group_finished(id) + return unless @examples_count >= threshold + + # Context-time + group_time = (TestProf.now - @group_started_at) - @examples_time + run_time = @examples_time + group_time + + groups << { + id: id, + run_time: run_time, + group_time: group_time, + count: @examples_count, + tps: -(@examples_count / run_time).round(2) + } + end + + def example_started(id) + @example_started_at = TestProf.now + end + + def example_finished(id) + @examples_count += 1 + @total_count += 1 + + time = (TestProf.now - @example_started_at) + @examples_time += time + @total_time += time + end + end + end +end diff --git a/lib/test_prof/tps_prof/reporter/text.rb b/lib/test_prof/tps_prof/reporter/text.rb new file mode 100644 index 00000000..96e284c4 --- /dev/null +++ b/lib/test_prof/tps_prof/reporter/text.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require "test_prof/ext/float_duration" +require "test_prof/ext/string_truncate" + +module TestProf + module TPSProf + module Reporter + class Text + include Logging + + using FloatDuration + using StringTruncate + + def print(profiler) + groups = profiler.groups + + total_tps = (profiler.total_count / profiler.total_time).round(2) + + msgs = [] + + msgs << + <<~MSG + Total TPS (tests per second): #{total_tps} + + Top #{profiler.top_count} slowest suites by TPS (tests per second) (min examples per group: #{profiler.threshold}): + + MSG + + groups.each do |group| + description = group[:id].top_level_description + location = group[:id].metadata[:location] + time = group[:run_time] + group_time = group[:group_time] + count = group[:count] + tps = -group[:tps] + + msgs << + <<~GROUP + #{description.truncate} (#{location}) – #{tps} TPS (#{time.duration} / #{count}), group time: #{group_time.duration} + GROUP + end + + log :info, msgs.join + end + end + end + end +end diff --git a/lib/test_prof/tps_prof/rspec.rb b/lib/test_prof/tps_prof/rspec.rb new file mode 100644 index 00000000..457dd223 --- /dev/null +++ b/lib/test_prof/tps_prof/rspec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module TestProf + module TPSProf + class RSpecListener # :nodoc: + include Logging + + NOTIFICATIONS = %i[ + example_group_started + example_group_finished + example_started + example_finished + ].freeze + + attr_reader :reporter, :profiler + + def initialize + @profiler = Profiler.new(TPSProf.config.top_count, threshold: TPSProf.config.threshold) + @reporter = TPSProf.config.reporter + + log :info, "TPSProf enabled (top-#{TPSProf.config.top_count})" + end + + def example_group_started(notification) + return unless notification.group.top_level? + profiler.group_started notification.group + end + + def example_group_finished(notification) + return unless notification.group.top_level? + profiler.group_finished notification.group + end + + def example_started(notification) + profiler.example_started notification.example + end + + def example_finished(notification) + profiler.example_finished notification.example + end + + def print + reporter.print(profiler) + end + end + end +end + +# Register TPSProf listener +TestProf.activate("TPS_PROF") do + RSpec.configure do |config| + listener = nil + + config.before(:suite) do + listener = TestProf::TPSProf::RSpecListener.new + config.reporter.register_listener( + listener, *TestProf::TPSProf::RSpecListener::NOTIFICATIONS + ) + end + + config.after(:suite) { listener&.print } + end +end diff --git a/lib/test_prof/utils/sized_ordered_set.rb b/lib/test_prof/utils/sized_ordered_set.rb index 8af86e80..81b00eeb 100644 --- a/lib/test_prof/utils/sized_ordered_set.rb +++ b/lib/test_prof/utils/sized_ordered_set.rb @@ -53,6 +53,10 @@ def size data.size end + def empty? + size.zero? + end + def to_a data.dup end diff --git a/lib/test_prof/vernier.rb b/lib/test_prof/vernier.rb new file mode 100644 index 00000000..4cac154a --- /dev/null +++ b/lib/test_prof/vernier.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module TestProf + # Vernier wrapper. + # + # Has 2 modes: global and per-example. + # + # Example: + # + # # To activate global profiling you can use env variable + # TEST_VERNIER=1 rspec ... + # + # To profile a specific examples add :vernier tag to it: + # + # it "is doing heavy stuff", :vernier do + # ... + # end + # + module Vernier + # Vernier configuration + class Configuration + attr_accessor :mode, :target, :interval, :hooks + + def initialize + @mode = ENV.fetch("TEST_VERNIER_MODE", :wall).to_sym + @target = (ENV["TEST_VERNIER"] == "boot") ? :boot : :suite + + sample_interval = ENV["TEST_VERNIER_INTERVAL"].to_i + @interval = (sample_interval > 0) ? sample_interval : nil + @hooks = ENV["TEST_VERNIER_HOOKS"]&.split(",")&.map { |hook| hook.strip.to_sym } + end + + def boot? + target == :boot + end + + def suite? + target == :suite + end + end + + class << self + include Logging + + def config + @config ||= Configuration.new + end + + def configure + yield config + end + + attr_reader :default_collector + + # Run Vernier and automatically dump + # a report when the process exits or when the application is booted. + def run + collector = profile + return unless collector + + @locked = true + @default_collector = collector + + log :info, "Vernier enabled globally: " \ + "mode – #{config.mode}, target – #{config.target}" + + at_exit { dump(collector, "total") } if config.suite? + end + + def profile(name = nil) + if locked? + log :warn, <<~MSG + Vernier has been already activated. + + Make sure you do not have the TEST_VERNIER environmental variable set somewhere. + MSG + + return false + end + + return false unless init_vernier + + options = {} + + options[:interval] = config.interval if config.interval + options[:hooks] = config.hooks if config.hooks + + if block_given? + options[:mode] = config.mode + options[:out] = build_path(name) + ::Vernier.trace(**options) { yield } + else + collector = ::Vernier::Collector.new(config.mode, **options) + collector.start + + collector + end + end + + def dump(collector, name) + result = collector.stop + + path = build_path(name) + + File.write(path, ::Vernier::Output::Firefox.new(result).output) + + log :info, "Vernier report generated: #{path}" + end + + private + + def build_path(name) + TestProf.artifact_path( + "vernier-report-#{config.mode}-#{name}.json" + ) + end + + def locked? + @locked == true + end + + def init_vernier + return @initialized if instance_variable_defined?(:@initialized) + @locked = false + @initialized = TestProf.require( + "vernier", + <<~MSG + Please, install 'vernier' first: + # Gemfile + gem 'vernier', '>= 0.3.0', require: false + MSG + ) { check_vernier_version } + end + + def check_vernier_version + if Utils.verify_gem_version("vernier", at_least: "0.3.0") + true + else + log :error, <<~MSG + Please, upgrade 'vernier' to version >= 0.3.0. + MSG + false + end + end + end + end +end + +require "test_prof/vernier/rspec" if TestProf.rspec? + +# Hook to run Vernier globally +TestProf.activate("TEST_VERNIER") do + TestProf::Vernier.run +end diff --git a/lib/test_prof/vernier/rspec.rb b/lib/test_prof/vernier/rspec.rb new file mode 100644 index 00000000..51148094 --- /dev/null +++ b/lib/test_prof/vernier/rspec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "test_prof/utils/rspec" + +module TestProf + module Vernier + # Reporter for RSpec to profile specific examples with Vernier + class Listener # :nodoc: + class << self + attr_accessor :report_name_generator + end + + self.report_name_generator = Utils::RSpec.method(:example_to_filename) + + NOTIFICATIONS = %i[ + example_started + example_finished + ].freeze + + def example_started(notification) + return unless profile?(notification.example) + notification.example.metadata[:vernier_collector] = TestProf::Vernier.profile + end + + def example_finished(notification) + return unless profile?(notification.example) + return unless notification.example.metadata[:vernier_collector] + + TestProf::Vernier.dump( + notification.example.metadata[:vernier_collector], + self.class.report_name_generator.call(notification.example) + ) + end + + private + + def profile?(example) + example.metadata.key?(:vernier) + end + end + end +end + +RSpec.configure do |config| + config.before(:suite) do + listener = TestProf::Vernier::Listener.new + + config.reporter.register_listener( + listener, *TestProf::Vernier::Listener::NOTIFICATIONS + ) + end +end + +# Handle boot profiling +RSpec.configure do |config| + config.append_before(:suite) do + TestProf::Vernier.dump(TestProf::Vernier.default_collector, "boot") if TestProf::Vernier.config.boot? + end +end diff --git a/lib/test_prof/version.rb b/lib/test_prof/version.rb index bc3d4189..6a327bcb 100644 --- a/lib/test_prof/version.rb +++ b/lib/test_prof/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module TestProf - VERSION = "1.0.6" + VERSION = "1.5.2" end diff --git a/spec/bugs/before_all_any_fixture_multidb_spec.rb b/spec/bugs/before_all_any_fixture_multidb_spec.rb new file mode 100644 index 00000000..a57e176f --- /dev/null +++ b/spec/bugs/before_all_any_fixture_multidb_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# https://github.com/test-prof/test-prof/issues/310 +describe "before_all + any_fixture + multidb", type: :integration do + specify "works" do + output = run_rspec( + "before_all_any_fixture_multidb", + chdir: File.join(__dir__, "fixtures") + ) + + expect(output).to include("0 failures") + end +end diff --git a/spec/bugs/fixtures/before_all_any_fixture_multidb_fixture.rb b/spec/bugs/fixtures/before_all_any_fixture_multidb_fixture.rb new file mode 100644 index 00000000..f087133a --- /dev/null +++ b/spec/bugs/fixtures/before_all_any_fixture_multidb_fixture.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require "action_controller/railtie" +require "action_view/railtie" +require "active_record/railtie" +require "rspec/rails" + +require_relative "../../support/ar_models" + +RSpec.configure do |config| + if config.respond_to?(:fixture_paths) + config.fixture_paths = [File.join(__dir__, "fixtures")] + else + config.fixture_path = File.join(__dir__, "fixtures") + end + config.use_transactional_fixtures = true +end + +require "test_prof/recipes/rspec/before_all" +require "test_prof/recipes/rspec/let_it_be" +require "test_prof/recipes/rspec/any_fixture" + +RSpec.configure do |config| + config.include TestProf::FactoryBot::Syntax::Methods + + config.before(:suite) do + TestProf::AnyFixture.register(:user) { TestProf::FactoryBot.create(:user) } + end +end + +# ActiveSupport::Notifications.subscribe("transaction.active_record") do |event| +# puts "[TRANSACTION] #{event.payload[:outcome].upcase} #{event.payload[:connection].inspect} from: #{caller_locations.find { |l| l.to_s.include?(Rails.root.to_s) }&.to_s}" +# end + +# TestProf::BeforeAll.configure do |config| +# config.before(:begin) do +# puts "[BEFORE_ALL] connection pools #{::ActiveRecord::Base.connection_handler.connection_pool_list(:writing).map(&:inspect)}" +# end + +# config.before(:rollback) do +# puts "[BEFORE_ALL] connection pools before unpin_connection! #{::ActiveRecord::Base.connection_handler.connection_pool_list(:writing).map(&:inspect)}" +# end +# end + +describe "let_it_be vs lazy multi db" do + let_it_be(:user) { TestProf::FactoryBot.create(:user) } + + # Loading an AR class with a custom DB configuration + # triggers a new connection pool creation that hasn't been tracked by + # before_all... + let!(:dual_comments_class) do + Class.new(ApplicationRecord) do + def self.name + "DualCommentsRecord" + end + + self.abstract_class = true + + connects_to database: {writing: :comments, reading: :primary} if multi_db? + end.then do |record_class| + Class.new(record_class) do + self.table_name = "comments" + + belongs_to :user, dependent: :destroy + end + end + end + + specify do + expect(User.count).to eq 2 + + dual_comments_class.create!(user: user, comment: "Hello!") + end + + specify do + expect(TestProf::FactoryBot.create(:comment)).to be_present + expect(Comment.count).to eq 1 + end + + context "with clean fixture", :with_clean_fixture do + specify { expect(User.count).to eq 0 } + end +end diff --git a/spec/bugs/fixtures/let_it_be_savepoint_fixture.rb b/spec/bugs/fixtures/let_it_be_savepoint_fixture.rb new file mode 100644 index 00000000..0136265f --- /dev/null +++ b/spec/bugs/fixtures/let_it_be_savepoint_fixture.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require "action_controller/railtie" +require "action_view/railtie" +require "active_record/railtie" +require "rspec/rails" + +require_relative "../../support/ar_models" + +RSpec.configure do |config| + if config.respond_to?(:fixture_paths) + config.fixture_paths = [File.join(__dir__, "fixtures")] + else + config.fixture_path = File.join(__dir__, "fixtures") + end + config.use_transactional_fixtures = true +end + +require "test_prof/recipes/rspec/let_it_be" + +RSpec.configure do |config| + config.include TestProf::FactoryBot::Syntax::Methods +end + +describe "let_it_be no-op" do + context "without db calls" do + let_it_be(:foo) { 1 } + + specify { expect(1).to eq 1 } + specify { expect(foo).to eq 1 } + end +end diff --git a/spec/bugs/fixtures/time_patch_fixture.rb b/spec/bugs/fixtures/time_patch_fixture.rb index 3e7d00c3..c51ea9d4 100644 --- a/spec/bugs/fixtures/time_patch_fixture.rb +++ b/spec/bugs/fixtures/time_patch_fixture.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require "active_support" require "active_support/testing/time_helpers" require "test-prof" diff --git a/spec/bugs/fixtures/timecop_fixture.rb b/spec/bugs/fixtures/timecop_fixture.rb index 15353285..251b24bd 100644 --- a/spec/bugs/fixtures/timecop_fixture.rb +++ b/spec/bugs/fixtures/timecop_fixture.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) -require "timecop" +require "timecop" if ENV.fetch("TIMECOP_ORDER", "before") == "before" require "test-prof" +require "timecop" if ENV.fetch("TIMECOP_ORDER", "before") == "after" Timecop.freeze diff --git a/spec/bugs/let_it_be_savepoint_spec.rb b/spec/bugs/let_it_be_savepoint_spec.rb new file mode 100644 index 00000000..9015df51 --- /dev/null +++ b/spec/bugs/let_it_be_savepoint_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# https://github.com/test-prof/test-prof/issues/265 +describe "let_it_be + mysql + savepoints", type: :integration do + specify "works" do + output = run_rspec( + "let_it_be_savepoint", + chdir: File.join(__dir__, "fixtures") + ) + + expect(output).to include("0 failures") + end +end diff --git a/spec/bugs/time_patch_handling_spec.rb b/spec/bugs/time_patch_handling_spec.rb index 66cbe371..6977a805 100644 --- a/spec/bugs/time_patch_handling_spec.rb +++ b/spec/bugs/time_patch_handling_spec.rb @@ -16,11 +16,24 @@ expect(s + ms).to be > 0 end - specify "works with timecop" do + specify "works with timecop loaded before test-prof" do output = run_rspec( "timecop", chdir: File.join(__dir__, "fixtures"), - env: {"TAG_PROF" => "a"} + env: {"TAG_PROF" => "a", "TIMECOP_ORDER" => "before"} + ) + + matches = output.match(/x\s+\d{2}:(\d{2})\.(\d{3})\s+/) + s, ms = matches[1].to_i, matches[2].to_i + + expect(s + ms).to be > 0 + end + + specify "works with timecop loaded after test-prof" do + output = run_rspec( + "timecop", + chdir: File.join(__dir__, "fixtures"), + env: {"TAG_PROF" => "a", "TIMECOP_ORDER" => "after"} ) matches = output.match(/x\s+\d{2}:(\d{2})\.(\d{3})\s+/) diff --git a/spec/cop_helper.rb b/spec/cop_helper.rb index fa16c66b..e8a66810 100644 --- a/spec/cop_helper.rb +++ b/spec/cop_helper.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require "rubocop" -require "test_prof/cops/inject" +require "rubocop/test_prof/plugin" require "rubocop/rspec/support" RSpec.configure do |config| diff --git a/spec/integrations/before_all_spec.rb b/spec/integrations/before_all_spec.rb index 7c69bcd9..c29a2e6d 100644 --- a/spec/integrations/before_all_spec.rb +++ b/spec/integrations/before_all_spec.rb @@ -27,6 +27,24 @@ expect(output).not_to include("SampleJob") expect(output).to include("FailingJob") end + + it "works with Rails fixtures" do + output = run_rspec("before_all_rails_fixtures", success: true) + + expect(output).to include("examples, 0 failures") + end + + specify "database connection" do + output = run_rspec("before_all_connection") + + expect(output).to include("0 failures") + end + + specify "dry-run" do + output = run_rspec("before_all", options: "--dry-run", env: {"DRY_RUN" => "true", "DB" => "postgres", "DATABASE_URL" => "postgres://bla-bla-host/test_prof_test"}) + + expect(output).to include("0 failures") + end end context "Minitest" do @@ -47,5 +65,11 @@ expect(output).to include("0 failures, 0 errors, 0 skips") end + + specify "database connection" do + output = run_minitest("before_all_connection") + + expect(output).to include("0 failures, 0 errors, 0 skip") + end end end diff --git a/spec/integrations/event_prof_minitest_spec.rb b/spec/integrations/event_prof_minitest_spec.rb index fd276248..895c3538 100644 --- a/spec/integrations/event_prof_minitest_spec.rb +++ b/spec/integrations/event_prof_minitest_spec.rb @@ -14,10 +14,10 @@ expect(output).to match(/Something \(\.\/event_prof_fixture\.rb\) – 00:00\.214 \(7 \/ 3\) of 00:00\.7\d{2} \(\d{2}\.\d+%\)/) expect(output).to match(/Another something \(\.\/event_prof_fixture\.rb\) – 00:00\.300 \(3 \/ 2\) of 00:00\.3\d{2} \(\d{2}.\d+%\)/) - expect(output).to match(/do very long ...nvokes 3 times \(\.\/event_prof_fixture\.rb:49\) – 00:00\.300 \(3\) of 00:00\.3\d{2} \(\d{2}\.\d+%\)/) - expect(output).to match(/invokes many times \(\.\/event_prof_fixture\.rb:35\) – 00:00\.136 \(4\) of 00:00\.4\d{2} \(\d{2}.\d+%\)/) - expect(output).to match(/invokes once \(\.\/event_prof_fixture\.rb:24\) – 00:00\.040 \(1\) of 00:00\.1\d{2} \(\d{2}\.\d+%\)/) - expect(output).to match(/invokes twice \(\.\/event_prof_fixture\.rb:29\) – 00:00\.038 \(2\) of 00:00\.2\d{2} \(\d{1,2}.\d+%\)/) + expect(output).to match(/do very long ...nvokes 3 times \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.300 \(3\) of 00:00\.3\d{2} \(\d{2}\.\d+%\)/) + expect(output).to match(/invokes many times \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.136 \(4\) of 00:00\.4\d{2} \(\d{2}.\d+%\)/) + expect(output).to match(/invokes once \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.040 \(1\) of 00:00\.1\d{2} \(\d{2}\.\d+%\)/) + expect(output).to match(/invokes twice \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.038 \(2\) of 00:00\.2\d{2} \(\d{1,2}.\d+%\)/) end specify "Minitest integration with rank by count", :aggregate_failures do output = run_minitest( @@ -35,10 +35,10 @@ expect(output).to match(/Something \(\.\/event_prof_fixture\.rb\) – 00:00\.214 \(7 \/ 3\) of 00:00\.7\d{2} \(\d{2}\.\d+%\)/) expect(output).to match(/Another something \(\.\/event_prof_fixture\.rb\) – 00:00\.300 \(3 \/ 2\) of 00:00\.3\d{2} \(\d{2}.\d+%\)/) - expect(output).to match(/do very long ...nvokes 3 times \(\.\/event_prof_fixture\.rb:49\) – 00:00\.300 \(3\) of 00:00\.3\d{2} \(\d{2}\.\d+%\)/) - expect(output).to match(/invokes many times \(\.\/event_prof_fixture\.rb:35\) – 00:00\.136 \(4\) of 00:00\.4\d{2} \(\d{2}.\d+%\)/) - expect(output).to match(/invokes once \(\.\/event_prof_fixture\.rb:24\) – 00:00\.040 \(1\) of 00:00\.1\d{2} \(\d{2}\.\d+%\)/) - expect(output).to match(/invokes twice \(\.\/event_prof_fixture\.rb:29\) – 00:00\.038 \(2\) of 00:00\.2\d{2} \(\d{1,2}.\d+%\)/) + expect(output).to match(/do very long ...nvokes 3 times \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.300 \(3\) of 00:00\.3\d{2} \(\d{2}\.\d+%\)/) + expect(output).to match(/invokes many times \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.136 \(4\) of 00:00\.4\d{2} \(\d{2}.\d+%\)/) + expect(output).to match(/invokes once \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.040 \(1\) of 00:00\.1\d{2} \(\d{2}\.\d+%\)/) + expect(output).to match(/invokes twice \(\.\/event_prof_fixture\.rb:\d+\) – 00:00\.038 \(2\) of 00:00\.2\d{2} \(\d{1,2}.\d+%\)/) end context "CustomEvents" do diff --git a/spec/integrations/factory_default_spec.rb b/spec/integrations/factory_default_spec.rb index b773da3b..bf87f960 100644 --- a/spec/integrations/factory_default_spec.rb +++ b/spec/integrations/factory_default_spec.rb @@ -1,9 +1,64 @@ # frozen_string_literal: true -describe "FactoryDefault" do - specify "RSpec integration", :aggregate_failures do - output = run_rspec("factory_default") +describe "FactoryDefault", :aggregate_failures do + context "RSpec integration" do + specify "basic" do + output = run_rspec("factory_default") - expect(output).to include("0 failures") + expect(output).to include("0 failures") + end + + specify "fabrication" do + output = run_rspec("factory_default_fabrication") + + expect(output).to include("0 failures") + end + + specify "let_it_be integration" do + output = run_rspec("factory_default_let_it_be") + + expect(output).to include("0 failures") + end + + specify "stats" do + output = run_rspec("factory_default", env: {"FACTORY_DEFAULT_STATS" => "1"}) + + expect(output).to include("0 failures") + + expect(output).to include("FactoryDefault summary: hit=11 miss=3") + expect(output).to match(/factory\s+hit\s+miss\n\n/) + expect(output).to match(/user\s+11\s+3/) + end + + specify "fabrication stats" do + output = run_rspec("factory_default_fabrication", env: {"FACTORY_DEFAULT_STATS" => "1"}) + + expect(output).to include("0 failures") + + expect(output).to include("FactoryDefault summary: hit=7 miss=1") + expect(output).to match(/factory\s+hit\s+miss\n\n/) + expect(output).to match(/user\s+7\s+1/) + end + + specify "analyze" do + output = run_rspec( + "factory_default_analyze", + env: { + "FACTORY_DEFAULT_PROF" => "1" + } + ) + + expect(output).to include("0 failures") + + expect(output).to include("Factory associations usage:") + expect(output).to match(/factory\s+count\s+total time\n\n/) + expect(output).to match(/user\s+7\s+\d{2}:\d{2}\.\d{3}/) + expect(output).to match(/user\[traited\]\s+1\s+\d{2}:\d{2}\.\d{3}/) + expect(output).to match(/user\{tag:"some tag"\}\s+1\s+\d{2}:\d{2}\.\d{3}/) + expect(output).to include("Total associations created: 9") + expect(output).to include("Total uniq associations created: 3") + expect(output).to match(/Total time spent: \d{2}:\d{2}\.\d{3}/) + expect(output).to include("0 failures") + end end end diff --git a/spec/integrations/factory_doctor_spec.rb b/spec/integrations/factory_doctor_spec.rb index 619efc79..b3950a10 100644 --- a/spec/integrations/factory_doctor_spec.rb +++ b/spec/integrations/factory_doctor_spec.rb @@ -10,16 +10,16 @@ expect(output).to match(/Total wasted time: \d{2}:\d{2}\.\d{3}/) expect(output).to include("User (./factory_doctor_fixture.rb)") - expect(output).to include("generates random names (./factory_doctor_fixture.rb:13) – 2 records created") - expect(output).to include("validates name (./factory_doctor_fixture.rb:18) – 1 record created") - expect(output).to include("clones (./factory_doctor_fixture.rb:28) – 1 record created") + expect(output).to include(%r{generates random names \(./factory_doctor_fixture.rb:\d+\) – 2 records created}) + expect(output).to include(%r{validates name \(./factory_doctor_fixture.rb:\d+\) – 1 record created}) + expect(output).to include(%r{clones \(./factory_doctor_fixture.rb:\d+\) – 1 record created}) expect(output).to include("PlainMinitestFabricationTest (./factory_doctor_fixture.rb)") expect(output).not_to include("is ignored") expect(output).not_to include("creates and reloads user") end it "print message when no bad examples", :aggregate_failures do - output = run_minitest("factory_doctor", env: {"FDOC" => "1", "FDOC_THRESHOLD" => "0", "TESTOPTS" => "--name=test_0005_is_ignored"}) + output = run_minitest("factory_doctor", env: {"FDOC" => "1", "FDOC_THRESHOLD" => "0", "TESTOPTS" => "--name=\"test_0005_is ignored\""}) expect(output).to include("FactoryDoctor enabled") expect(output).to include('FactoryDoctor says: "Looks good to me!"') @@ -34,10 +34,10 @@ expect(output).to include("Total (potentially) bad examples: 4") expect(output).to match(/Total wasted time: \d{2}:\d{2}\.\d{3}/) - expect(output).to include("User (./factory_doctor_fixture.rb:7)") - expect(output).to include("generates random names (./factory_doctor_fixture.rb:10) – 2 records created") - expect(output).to include("validates name (./factory_doctor_fixture.rb:15) – 1 record created") - expect(output).to include("clones (./factory_doctor_fixture.rb:25) – 1 record created") + expect(output).to include(%r{User \(./factory_doctor_fixture.rb:\d+\)}) + expect(output).to include(%r{generates random names \(./factory_doctor_fixture.rb:\d+\) – 2 records created}) + expect(output).to include(%r{validates name \(./factory_doctor_fixture.rb:\d+\) – 1 record created}) + expect(output).to include(%r{clones \(./factory_doctor_fixture.rb:\d+\) – 1 record created}) expect(output).not_to include("is ignored") expect(output).not_to include("creates and reloads user") end diff --git a/spec/integrations/factory_prof_spec.rb b/spec/integrations/factory_prof_spec.rb index 727c6a4f..ff8f55ff 100644 --- a/spec/integrations/factory_prof_spec.rb +++ b/spec/integrations/factory_prof_spec.rb @@ -8,9 +8,95 @@ expect(output).to include("FactoryProf enabled (simple mode)") expect(output).to include("Factories usage") - expect(output).to match(/Total: 26\n\s+Total top-level: 14\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 2/) - expect(output).to match(/total\s+top-level\s+total time\s+time per call\s+top-level time\s+name/) - expect(output).to match(/\s+16\s+8\s+(\d+\.\d{4}s\s+){3}user\n\s+10\s+6\s+\s+(\d+\.\d{4}s\s+){3}post/) + expect(output).to match(/Total: 30\n\s+Total top-level: 18\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 3/) + expect(output).to match(/name\s+total\s+top-level\s+total time\s+time per call\s+top-level time/) + expect(output).to match( + / + user\s+16\s+8\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+post\s+10\s+6\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+supercalifragilisticexpialidocious\s+4\s+4\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + /x + ) + end + + specify "simple printer with variations", :aggregate_failures do + output = run_rspec("factory_prof_with_variations", env: {"FPROF" => "1", "FPROF_VARS" => "1", "FPROF_VARIATIONS_LIMIT" => "2"}) + + expect(output).to include("FactoryProf enabled (simple mode)") + + expect(output).to include("Factories usage") + expect(output).to match(/Total: 29\n\s+Total top-level: 13\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 3/) + expect(output).to match(/name\s+total\s+top-level\s+total time\s+time per call\s+top-level time/) + expect(output).to match( + / + user\s+15\s+7\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+-\s+9\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.traited.with_posts\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[name\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[...\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+post\s+10\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+-\s+8\s+0\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[text,\suser\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s + \s+supercalifragilisticexpialidocious\s+4\s+4\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.other_trait_with_very_long_name\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.traited\[tag\]\s+1\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[...\]\s+1\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + /x + ) + end + + specify "simple printer with threshold param", :aggregate_failures do + output = run_rspec("factory_prof", env: {"FPROF" => "1", "FPROF_THRESHOLD" => "11"}) + + expect(output).to include("FactoryProf enabled (simple mode)") + + expect(output).to include("Factories usage") + expect(output).to match(/Total: 30\n\s+Total top-level: 18\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 3/) + expect(output).to match(/name\s+total\s+top-level\s+total time\s+time per call\s+top-level time/) + expect(output).to match(/user\s+16\s+8\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n/) + expect(output).not_to match(/\s+post\s+10\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n/) + end + + specify "simple printer truncated", :aggregate_failures do + output = run_rspec("factory_prof", env: {"FPROF" => "1", "FPROF_TRUNCATE_NAMES" => "1"}) + expect(output).to include("FactoryProf enabled (simple mode)") + + expect(output).to include("Factories usage") + expect(output).to match(/Total: 30\n\s+Total top-level: 18\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 3/) + expect(output).to match(/name\s+total\s+top-level\s+total time\s+time per call\s+top-level time/) + expect(output).to match( + / + user\s+16\s+8\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+post\s+10\s+6\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+supercalifragilis...\s+4\s+4\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + /x + ) + end + + specify "simple printer truncated with variations", :aggregate_failures do + output = run_rspec("factory_prof_with_variations", env: {"FPROF" => "1", "FPROF_TRUNCATE_NAMES" => "1", "FPROF_VARS" => "1", "FPROF_VARIATIONS_LIMIT" => "2"}) + + expect(output).to include("FactoryProf enabled (simple mode)") + + expect(output).to include("Factories usage") + expect(output).to match(/Total: 29\n\s+Total top-level: 13\n\s+Total time: \d{2}+:\d{2}\.\d{3} \(out of \d{2}+:\d{2}\.\d{3}\)\n\s+Total uniq factories: 3/) + expect(output).to match(/name\s+total\s+top-level\s+total time\s+time per call\s+top-level time/) + expect(output).to match( + / + user\s+15\s+7\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+-\s+9\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.traited.with_p...\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[name\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[...\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+post\s+10\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+-\s+8\s+0\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[text,\suser\]\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s + \s+supercalifragilis...\s+4\s+4\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.other_trait_wi...\s+2\s+2\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+.traited\[tag\]\s+1\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + \s+\[...\]\s+1\s+1\s+(\d+\.\d{4}s\s+){2}\d+\.\d{4}s\n + /x + ) end specify "flamegraph printer" do @@ -43,6 +129,13 @@ expect(output).to include("FactoryFlame report generated: ") end + specify "with nate printer always enabled and json profiler", :aggregate_failures do + output = run_rspec("factory_prof_with_nate", env: {"FPROF" => "json"}) + + expect(output).to match(/Time spent in factories: \d{2}+:\d{2}\.\d{3} \([\d.]+% of total time\)/) + expect(output).to include("Profile results to JSON: ") + end + context "when no fabrication installed" do specify "simple printer", :aggregate_failures do output = run_rspec("factory_prof_no_fabrication", env: {"FPROF" => "1"}) diff --git a/spec/integrations/fixtures/minitest/before_all_connection_fixture.rb b/spec/integrations/fixtures/minitest/before_all_connection_fixture.rb new file mode 100644 index 00000000..ecad0914 --- /dev/null +++ b/spec/integrations/fixtures/minitest/before_all_connection_fixture.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_minitest" +require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + +require "test_prof/recipes/minitest/before_all" + +describe "database connection owner" do + describe "with before_all" do + include TestProf::BeforeAll::Minitest + + before_all {} + + it "uses the connection owned by main thread" do + main_thread = Thread.current + Thread.new do + assert_equal ActiveRecord::Base.connection.owner, main_thread + end.join + end + end + + describe "without before_all" do + it "uses the connection owned by each thread" do + Thread.new do + child_thread = Thread.current + assert_equal ActiveRecord::Base.connection.owner, child_thread + end.join + end + end +end diff --git a/spec/integrations/fixtures/minitest/before_all_fixture.rb b/spec/integrations/fixtures/minitest/before_all_fixture.rb index 2f3005a2..28346684 100644 --- a/spec/integrations/fixtures/minitest/before_all_fixture.rb +++ b/spec/integrations/fixtures/minitest/before_all_fixture.rb @@ -4,6 +4,7 @@ require_relative "../../../support/ar_models" require_relative "../../../support/transactional_minitest" require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) require "test_prof/recipes/minitest/before_all" diff --git a/spec/integrations/fixtures/minitest/before_all_inherit_fixture.rb b/spec/integrations/fixtures/minitest/before_all_inherit_fixture.rb index fd77d7f9..a4313cd8 100644 --- a/spec/integrations/fixtures/minitest/before_all_inherit_fixture.rb +++ b/spec/integrations/fixtures/minitest/before_all_inherit_fixture.rb @@ -4,6 +4,7 @@ require_relative "../../../support/ar_models" require_relative "../../../support/transactional_minitest" require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) require "test_prof/recipes/minitest/before_all" diff --git a/spec/integrations/fixtures/minitest/event_prof_factory_create_fixture.rb b/spec/integrations/fixtures/minitest/event_prof_factory_create_fixture.rb index 38854e70..7c813177 100644 --- a/spec/integrations/fixtures/minitest/event_prof_factory_create_fixture.rb +++ b/spec/integrations/fixtures/minitest/event_prof_factory_create_fixture.rb @@ -3,6 +3,8 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require_relative "../../../support/ar_models" require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + require "test-prof" describe "Post" do diff --git a/spec/integrations/fixtures/minitest/event_prof_fixture.rb b/spec/integrations/fixtures/minitest/event_prof_fixture.rb index f0ab0049..40200173 100644 --- a/spec/integrations/fixtures/minitest/event_prof_fixture.rb +++ b/spec/integrations/fixtures/minitest/event_prof_fixture.rb @@ -2,6 +2,8 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + require "active_support" require "test-prof" diff --git a/spec/integrations/fixtures/minitest/event_prof_sidekiq_fixture.rb b/spec/integrations/fixtures/minitest/event_prof_sidekiq_fixture.rb index fbd7bf7e..1833c2da 100644 --- a/spec/integrations/fixtures/minitest/event_prof_sidekiq_fixture.rb +++ b/spec/integrations/fixtures/minitest/event_prof_sidekiq_fixture.rb @@ -3,6 +3,8 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require "active_support" require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + require "sidekiq/testing" Sidekiq::Testing.inline! diff --git a/spec/integrations/fixtures/minitest/factory_doctor_fixture.rb b/spec/integrations/fixtures/minitest/factory_doctor_fixture.rb index 4c40517b..826e4328 100644 --- a/spec/integrations/fixtures/minitest/factory_doctor_fixture.rb +++ b/spec/integrations/fixtures/minitest/factory_doctor_fixture.rb @@ -3,6 +3,8 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require_relative "../../../support/ar_models" require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + require "test-prof" describe "User" do diff --git a/spec/integrations/fixtures/minitest/memory_prof_fixture.rb b/spec/integrations/fixtures/minitest/memory_prof_fixture.rb new file mode 100644 index 00000000..3efc1987 --- /dev/null +++ b/spec/integrations/fixtures/minitest/memory_prof_fixture.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "test-prof" +Minitest.load :test_prof if Minitest.respond_to?(:load) + +require "securerandom" + +describe "First Allocations" do + it "allocates 500 objects" do + 500.times.map { SecureRandom.hex } + end + + it "allocates 1000 objects" do + 1000.times.map { SecureRandom.hex } + end + + it "allocates 10_000 objects" do + 10_000.times.map { SecureRandom.hex } + end +end + +describe "Second Allocations" do + it "allocates nothing" do + end + + it "allocates 100 objects" do + 100.times.map { SecureRandom.hex } + end +end diff --git a/spec/integrations/fixtures/minitest/profilers_fixture.rb b/spec/integrations/fixtures/minitest/profilers_fixture.rb index e3757ca9..43959438 100644 --- a/spec/integrations/fixtures/minitest/profilers_fixture.rb +++ b/spec/integrations/fixtures/minitest/profilers_fixture.rb @@ -3,6 +3,7 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require "minitest/autorun" require "test_prof" +Minitest.load :test_prof if Minitest.respond_to?(:load) TestProf.configure do |config| config.output_dir = "../../../../tmp/test_prof" diff --git a/spec/integrations/fixtures/minitest/sample_fixture.rb b/spec/integrations/fixtures/minitest/sample_fixture.rb index dd0bec01..242ae80e 100644 --- a/spec/integrations/fixtures/minitest/sample_fixture.rb +++ b/spec/integrations/fixtures/minitest/sample_fixture.rb @@ -2,6 +2,8 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + require "test_prof/recipes/minitest/sample" class SomethingTest < Minitest::Test @@ -14,7 +16,10 @@ def test_pass2 end end -class AnotherSomethingTest < Minitest::Test +class CustomTestCase < Minitest::Test +end + +class AnotherSomethingTest < CustomTestCase def test_pass assert true end diff --git a/spec/integrations/fixtures/minitest/tag_prof_fixture.rb b/spec/integrations/fixtures/minitest/tag_prof_fixture.rb new file mode 100644 index 00000000..55cc7ad7 --- /dev/null +++ b/spec/integrations/fixtures/minitest/tag_prof_fixture.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require "minitest/autorun" +Minitest.load :test_prof if Minitest.respond_to?(:load) + +require "active_support" +require "test-prof" + +module Instrumenter + def self.notify(_event, time) + sleep 0.1 + ActiveSupport::Notifications.publish( + "test.event", + 0, + time + ) + end +end + +describe "Test Class" do + it "succeeds" do + Instrumenter.notify "test.event", 100 + assert(true) + end +end diff --git a/spec/integrations/fixtures/rspec/any_fixture_callbacks_fixture.rb b/spec/integrations/fixtures/rspec/any_fixture_callbacks_fixture.rb new file mode 100644 index 00000000..444ec46a --- /dev/null +++ b/spec/integrations/fixtures/rspec/any_fixture_callbacks_fixture.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require "test_prof/recipes/rspec/any_fixture" + +require "test_prof/any_fixture/dsl" + +shared_context "user_and_post", user_and_post: true do + include TestProf::AnyFixture::DSL + + before(:all) do + @user = any_fixture(:user) do + TestProf::FactoryBot.create(:user) + end + end + + before { TestProf::FactoryBot.create(:post) } + + let(:user) { User.find(fixture(:user).id) } +end + +describe "before_fixtures_reset callback", :user_and_post do + before(:all) do + before_any_fixtures_reset do + Post.delete_all + end + end + + it "deletes post" do + expect { TestProf::AnyFixture.reset }.to change(Post, :count).by(-1) + end +end + +describe "after_fixtures_reset callback", :user_and_post do + before(:all) do + after_any_fixtures_reset do + Post.delete_all + end + end + + it "deletes post" do + expect { TestProf::AnyFixture.reset }.to change(Post, :count).by(-1) + end +end + +describe "without callbacks", :user_and_post do + before { TestProf::FactoryBot.create(:post) } + after { Post.delete_all } + + it "doesn't delete post" do + expect { TestProf::AnyFixture.reset }.not_to change(Post, :count) + end +end diff --git a/spec/integrations/fixtures/rspec/any_fixture_fixture.rb b/spec/integrations/fixtures/rspec/any_fixture_fixture.rb index a28c03cd..90f79f96 100644 --- a/spec/integrations/fixtures/rspec/any_fixture_fixture.rb +++ b/spec/integrations/fixtures/rspec/any_fixture_fixture.rb @@ -5,17 +5,17 @@ require_relative "../../../support/transactional_context" require "test_prof/recipes/rspec/any_fixture" -require "test_prof/any_fixture/dsl" -using TestProf::AnyFixture::DSL - shared_context "user", user: true do + include TestProf::AnyFixture::DSL + before(:all) do @user = fixture(:user) do TestProf::FactoryBot.create(:user) end end - let(:user) { User.find(fixture(:user).id) } + let(:user) { fixture(:user) } + let(:a_user) { fixture(:user) { TestProf::FactoryBot.create(:user) } } end describe "User", :user do @@ -27,6 +27,7 @@ context "with clean fixture", :transactional do specify "no users", :with_clean_fixture do expect(User.count).to eq 0 + expect { a_user }.to change(User, :count).by(1) end end end diff --git a/spec/integrations/fixtures/rspec/before_all_connection_fixture.rb b/spec/integrations/fixtures/rspec/before_all_connection_fixture.rb new file mode 100644 index 00000000..5f33b5e4 --- /dev/null +++ b/spec/integrations/fixtures/rspec/before_all_connection_fixture.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" +require "test_prof/recipes/rspec/before_all" + +describe "database connection owner" do + context "with before_all" do + before_all {} + + it "uses the connection owned by main thread" do + main_thread = Thread.current + Thread.new do + expect(ActiveRecord::Base.connection.owner).to eq(main_thread) + end.join + end + end + + context "without before_all" do + it "uses the connection owned by each thread" do + Thread.new do + child_thread = Thread.current + expect(ActiveRecord::Base.connection.owner).to eq(child_thread) + end.join + end + end +end diff --git a/spec/integrations/fixtures/rspec/before_all_fixture.rb b/spec/integrations/fixtures/rspec/before_all_fixture.rb index 9f3f330d..d9123e75 100644 --- a/spec/integrations/fixtures/rspec/before_all_fixture.rb +++ b/spec/integrations/fixtures/rspec/before_all_fixture.rb @@ -78,6 +78,10 @@ specify { expect(User.count).to eq 1 } end + + context "after before_all with thread the database must be clean" do + specify { expect(User.count).to eq 0 } + end end describe "User", :transactional, :with_user do diff --git a/spec/integrations/fixtures/rspec/before_all_hooks_fixture.rb b/spec/integrations/fixtures/rspec/before_all_hooks_fixture.rb index a6d0dc37..514c8ed9 100644 --- a/spec/integrations/fixtures/rspec/before_all_hooks_fixture.rb +++ b/spec/integrations/fixtures/rspec/before_all_hooks_fixture.rb @@ -30,10 +30,30 @@ def add_event(event) Events.add_event :setup_before_all end + config.before(:begin, :with_meta) do + Events.add_event :setup_before_all_with_meta + end + + config.before(:begin, with_meta: false) do + Events.add_event :setup_before_all_with_meta_false + end + + config.before(:begin, foo: :bar) do + Events.add_event :setup_before_all_with_bar_tag + end + + config.before(:begin, foo: :baz) do + Events.add_event :setup_before_all_with_baz_tag + end + config.after(:begin) do Events.add_event :before_all_was_set_up end + config.after(:begin, with_meta: proc(&:present?)) do + Events.add_event :before_all_was_set_up_with_meta + end + config.before(:rollback) do # create user to check the it's created within a transaction hook_user = TestProf::FactoryBot.create(:user) @@ -82,4 +102,23 @@ def add_event(event) end end end + + context "with before_all" do + context "with matched metadata", with_meta: true, foo: :bar do + before_all { Events.add_event :before_all } + after(:all) { Events.events.clear } + + it "should setup before_all_with_meta" do + expect(Events.events).to eq([ + :setup_before_all, + :setup_before_all_with_meta, + :setup_before_all_with_bar_tag, + :before_all_was_set_up, + :before_all_was_set_up_with_meta, + :before_all, + :before_each + ]) + end + end + end end diff --git a/spec/integrations/fixtures/rspec/before_all_isolator_fixture.rb b/spec/integrations/fixtures/rspec/before_all_isolator_fixture.rb index ac831bca..23d6dc23 100644 --- a/spec/integrations/fixtures/rspec/before_all_isolator_fixture.rb +++ b/spec/integrations/fixtures/rspec/before_all_isolator_fixture.rb @@ -28,7 +28,7 @@ def perform(*_args) end end -class User < ActiveRecord::Base +class User < ApplicationRecord attr_accessor :commited end diff --git a/spec/integrations/fixtures/rspec/before_all_rails_fixtures_fixture.rb b/spec/integrations/fixtures/rspec/before_all_rails_fixtures_fixture.rb new file mode 100644 index 00000000..fd3306af --- /dev/null +++ b/spec/integrations/fixtures/rspec/before_all_rails_fixtures_fixture.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) + +require "action_controller/railtie" +require "action_view/railtie" +require "active_record/railtie" +require "rspec/rails" + +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" +require "test_prof/recipes/rspec/before_all" +require "test_prof/recipes/rspec/let_it_be" + +RSpec.configure do |config| + if config.respond_to?(:fixture_paths) + config.fixture_paths = [File.join(__dir__, "fixtures")] + else + config.fixture_path = File.join(__dir__, "fixtures") + end + config.use_transactional_fixtures = true +end + +TestProf::BeforeAll.configure do |config| + config.setup_fixtures = true +end + +describe "Post" do + fixtures :users + + context "with before_all" do + before_all do + @post = TestProf::FactoryBot.create(:post, text: "Test fixtures", user: users(:vova)) + end + + let(:post) { Post.find(@post.id) } + + it "text and user" do + expect(post.user).not_to be_nil + post.text = "" + post.user_id = nil + post.save! + expect(post.reload.text).to be_empty + expect(post.user).to be_nil + end + + it "old text and user" do + expect(post.text).to eq "Test fixtures" + expect(post.user.name).to eq "Vova" + end + end + + context "without before_all" do + specify "no posts" do + expect(Post.count).to eq 0 + end + end + + context "before_all with thread" do + before_all do + Thread.new do + TestProf::FactoryBot.create(:user, tag: :thread) + end.join + end + + specify { expect(User.find_by(tag: :thread)).to be_a(User) } + + specify { expect(User.count).to eq 2 } + end + + context "after before_all with thread the database must be clean" do + specify { expect(User.count).to eq 1 } + end +end diff --git a/spec/integrations/fixtures/rspec/event_prof_monitor_fixture.rb b/spec/integrations/fixtures/rspec/event_prof_monitor_fixture.rb index 91fef698..ae2ecc95 100644 --- a/spec/integrations/fixtures/rspec/event_prof_monitor_fixture.rb +++ b/spec/integrations/fixtures/rspec/event_prof_monitor_fixture.rb @@ -14,8 +14,8 @@ def one true end - def two - false + def two(flag: false) + flag end end @@ -28,6 +28,6 @@ def two end it "invokes twice" do - expect(w.one && w.two).to eq false + expect(w.two(flag: true) && w.two).to eq false end end diff --git a/spec/integrations/fixtures/rspec/factory_default_analyze_fixture.rb b/spec/integrations/fixtures/rspec/factory_default_analyze_fixture.rb new file mode 100644 index 00000000..0ca497a7 --- /dev/null +++ b/spec/integrations/fixtures/rspec/factory_default_analyze_fixture.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" +require "test_prof/recipes/rspec/factory_default" + +describe "Post" do + let(:user) { TestProf::FactoryBot.create(:user) } + let(:post) { TestProf::FactoryBot.create(:post) } + + it "creates post with different user" do + user + expect { post }.to change(User, :count) + expect(post.user).not_to eq user + end + + it "creates user if no default" do + expect { post }.to change(User, :count).by(1) + end + + it "creates many records" do + user + expect { TestProf::FactoryBot.create_list(:post, 5) }.to change(User, :count).by(5) + end + + context "with traits" do + let(:post) { TestProf::FactoryBot.create(:post, :with_traited_user) } + + it "can still be set default" do + expect(post.user.tag).to eq "traited" + end + end + + context "with overrides" do + let(:post) { TestProf::FactoryBot.create(:post, :with_tagged_user) } + + it "can still be set default" do + expect(post.user.tag).to eq "some tag" + end + end +end diff --git a/spec/integrations/fixtures/rspec/factory_default_fabrication_fixture.rb b/spec/integrations/fixtures/rspec/factory_default_fabrication_fixture.rb new file mode 100644 index 00000000..0ea22fef --- /dev/null +++ b/spec/integrations/fixtures/rspec/factory_default_fabrication_fixture.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" +require "test_prof/recipes/rspec/factory_default" + +TestProf::FactoryDefault.configure do |config| + config.preserve_attributes = true +end + +describe "Post" do + let(:user) { Fabricate.create_default(:user) } + let(:post) { Fabricate(:post) } + + it "creates post with the same user" do + user + expect { post }.not_to change(User, :count) + expect(post.user).to eq user + end + + it "creates associated user if no default" do + expect { post }.to change(User, :count).by(1) + end + + it "creates new user if not an association" do + user + expect { Fabricate(:user) }.to change(User, :count).by(1) + end + + it "works with many records" do + user + expect { Fabricate.times(5, :post) }.not_to change(User, :count) + expect(user.posts.count).to eq 5 + end + + it "works with specified user" do + user + user2 = Fabricate(:user) + post = Fabricate(:post, user: user2) + expect(post.user).to eq user2 + end + + context "with redefined user" do + let(:user2) { Fabricate(:user) } + + before { Fabricate.set_fabricate_default(:user, user2) } + + it "uses redefined default" do + expect(post.user).to eq user2 + end + end + + context "with overrides" do + before { user } + + it "creates new record if overrides do not match" do + expect { Fabricate(:alice_post) }.to change(User, :count).by(1) + end + end +end diff --git a/spec/integrations/fixtures/rspec/factory_default_fixture.rb b/spec/integrations/fixtures/rspec/factory_default_fixture.rb index 5e41b8ab..a4613d06 100644 --- a/spec/integrations/fixtures/rspec/factory_default_fixture.rb +++ b/spec/integrations/fixtures/rspec/factory_default_fixture.rb @@ -47,7 +47,11 @@ let(:traited_user) { TestProf::FactoryBot.create_default(:user, :traited, tag: "foo") } context "global setting" do - before { TestProf::FactoryDefault.preserve_traits = true } + before do + TestProf::FactoryDefault.configure do |config| + config.preserve_traits = true + end + end it "can still be set default" do expect(traited_user.tag).to eq "foo" @@ -66,7 +70,12 @@ end context "local override" do - before { TestProf::FactoryDefault.preserve_traits = false } + before do + TestProf::FactoryDefault.configure do |config| + config.preserve_traits = false + end + end + let(:override_user) { TestProf::FactoryBot.create_default(:user, preserve_traits: true) } let(:other_traited_post) { TestProf::FactoryBot.create(:post, :with_traited_user) } @@ -83,5 +92,18 @@ }.to change(User, :count).by(3) end end + + context "thread safety" do + let(:user) { TestProf::FactoryBot.create(:user) } + let(:post) { TestProf::FactoryBot.create(:post) } + + it "storing defaults is done per thread" do + user + + Thread.new { TestProf::FactoryBot.set_factory_default(:user, user) }.join + + expect(post.user).not_to eq user + end + end end end diff --git a/spec/integrations/fixtures/rspec/factory_default_let_it_be_fixture.rb b/spec/integrations/fixtures/rspec/factory_default_let_it_be_fixture.rb new file mode 100644 index 00000000..d795eb04 --- /dev/null +++ b/spec/integrations/fixtures/rspec/factory_default_let_it_be_fixture.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" + +require "test_prof/recipes/rspec/let_it_be" +require "test_prof/recipes/rspec/factory_default" + +describe "Post" do + let(:post) { TestProf::FactoryBot.create(:post) } + + context "with let_it_be" do + let_it_be(:user) { TestProf::FactoryBot.create_default(:user) } + + it "creates post with the same user" do + user + expect { post }.not_to change(User, :count) + expect(post.user).to eq user + end + + it "still uses the default from let_it_be" do + expect { post }.not_to change(User, :count) + end + + context "when nested" do + let_it_be(:post) { TestProf::FactoryBot.create_default(:post) } + + it "still uses the default from let_it_be" do + expect { post }.not_to change(User, :count) + end + + it "default is used within let_it_be" do + expect(post.user).to eq user + end + end + end + + context "without let_it_be" do + it "creates a new record" do + expect { post }.to change(User, :count).by(1) + end + end +end diff --git a/spec/integrations/fixtures/rspec/factory_doctor_fixture.rb b/spec/integrations/fixtures/rspec/factory_doctor_fixture.rb index df20e3c6..1a9f0a93 100644 --- a/spec/integrations/fixtures/rspec/factory_doctor_fixture.rb +++ b/spec/integrations/fixtures/rspec/factory_doctor_fixture.rb @@ -35,8 +35,9 @@ let(:user) { Fabricate(:user) } it "creates and reloads user" do - user = Fabricate(:user) - expect(User.find(user.id).name).to eq "John 1" + user + user2 = Fabricate(:user) + expect(User.find(user2.id).name).to eq "John 1" end it "validates name" do diff --git a/spec/integrations/fixtures/rspec/factory_prof_fixture.rb b/spec/integrations/fixtures/rspec/factory_prof_fixture.rb index 7004a83b..c1897389 100644 --- a/spec/integrations/fixtures/rspec/factory_prof_fixture.rb +++ b/spec/integrations/fixtures/rspec/factory_prof_fixture.rb @@ -71,3 +71,25 @@ end end end + +describe "Supercalifragilisticexpialidocious" do + let(:factory) { :supercalifragilisticexpialidocious } + + context "created by factory_bot" do + let(:supercali) { TestProf::FactoryBot.create(factory) } + + it "generates random names" do + supercali2 = TestProf::FactoryBot.create(factory) + expect(supercali.name).not_to eq supercali2.name + end + end + + context "created by fabrication" do + let(:supercali) { Fabricate(factory) } + + it "generates random names" do + supercali2 = Fabricate(factory) + expect(supercali.name).not_to eq supercali2.name + end + end +end diff --git a/spec/integrations/fixtures/rspec/factory_prof_no_factory_bot_fixture.rb b/spec/integrations/fixtures/rspec/factory_prof_no_factory_bot_fixture.rb index 714aedd0..3efbf780 100644 --- a/spec/integrations/fixtures/rspec/factory_prof_no_factory_bot_fixture.rb +++ b/spec/integrations/fixtures/rspec/factory_prof_no_factory_bot_fixture.rb @@ -2,13 +2,12 @@ $LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) -$LOAD_PATH.delete_if { |p| (p =~ /factory_girl/) || (p =~ /factory_bot/) } +$LOAD_PATH.delete_if { |p| p =~ /factory_bot/ } require "test-prof" context "when no factory_bot installed" do it "do nothing" do - expect { FactoryGirl.create(:user) }.to raise_error(NameError) expect { FactoryBot.create(:user) }.to raise_error(NameError) expect(true).to eq true end diff --git a/spec/integrations/fixtures/rspec/factory_prof_with_variations_fixture.rb b/spec/integrations/fixtures/rspec/factory_prof_with_variations_fixture.rb new file mode 100644 index 00000000..0097f016 --- /dev/null +++ b/spec/integrations/fixtures/rspec/factory_prof_with_variations_fixture.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require "test-prof" + +TestProf.configure do |config| + config.output_dir = "../../../../tmp/test_prof" +end + +describe "User" do + context "created by factory_bot" do + context "with few traits" do + let!(:user_with_traits) { TestProf::FactoryBot.create(:user, :traited, :with_posts) } + let!(:user_with_same_traits) { TestProf::FactoryBot.create(:user, :with_posts, :traited) } + + it "works" do + expect(true).to eq true + end + end + + context "with many traits" do + let!(:user_over_limit) { TestProf::FactoryBot.create(:user, :with_posts, :traited, :other_trait, tag: "tag") } + let!(:another_user_over_limit) { TestProf::FactoryBot.create(:user, :with_posts, :traited, tag: "some tag") } + + it "works" do + expect(true).to eq true + end + end + end + + context "created by fabrication" do + let!(:user) { Fabricate(:user) } + let!(:another_user_1) { Fabricate(:user, name: "some name") } + let!(:another_user_2) { Fabricate(:user, name: "some name") } + let!(:post) { Fabricate(:post, user: user, text: "some text") } + let!(:post_with_same_overrides) { Fabricate(:post, text: "some text", user: user) } + + it "works" do + expect(true).to eq true + end + end +end + +describe "Supercalifragilisticexpialidocious" do + let(:factory) { :supercalifragilisticexpialidocious } + let(:trait) { :other_trait_with_very_long_name } + let(:other_trait) { :traited } + + context "created by factory_bot" do + context "with few traits" do + let!(:super_with_traits) { TestProf::FactoryBot.create(factory, trait) } + let!(:super_with_same_traits) { TestProf::FactoryBot.create(factory, trait) } + + it "works" do + expect(true).to eq true + end + end + + context "with many traits" do + let!(:super_over_limit) { TestProf::FactoryBot.create(factory, trait, other_trait, tag: "tag") } + let!(:another_super_over_limit) { TestProf::FactoryBot.create(factory, other_trait, tag: "some tag") } + + it "works" do + expect(true).to eq true + end + end + end +end diff --git a/spec/integrations/fixtures/rspec/fixtures/users.yml b/spec/integrations/fixtures/rspec/fixtures/users.yml new file mode 100644 index 00000000..d4b1e1ec --- /dev/null +++ b/spec/integrations/fixtures/rspec/fixtures/users.yml @@ -0,0 +1,3 @@ +vova: + name: "Vova" + tag: "martian" diff --git a/spec/integrations/fixtures/rspec/let_it_be_fixture.rb b/spec/integrations/fixtures/rspec/let_it_be_fixture.rb index 4f51f708..0bc6dc21 100644 --- a/spec/integrations/fixtures/rspec/let_it_be_fixture.rb +++ b/spec/integrations/fixtures/rspec/let_it_be_fixture.rb @@ -40,6 +40,7 @@ let_it_be(:user) { create(:user) } before(:all) { @cache[:user_name] = user.name } + after(:all) { expect(user.name).to eq @cache[:user_name] } it "is cached" do expect(user.name).to eq @cache[:user_name] @@ -90,7 +91,7 @@ end context "with refind option" do - Π΄Π°_Π±ΡƒΠ΄Π΅Ρ‚_Ρ‚Π°ΠΊ(:post, refind: true) { create(:post) } # rubocop:disable Naming/AsciiIdentifiers + Π΄Π°_Π±ΡƒΠ΄Π΅Ρ‚_Ρ‚Π°ΠΊ(:post, refind: true) { create(:post) } let(:user) { post.user } diff --git a/spec/integrations/fixtures/rspec/let_it_be_nested_fixture.rb b/spec/integrations/fixtures/rspec/let_it_be_nested_fixture.rb new file mode 100644 index 00000000..57868877 --- /dev/null +++ b/spec/integrations/fixtures/rspec/let_it_be_nested_fixture.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require_relative "../../../support/ar_models" +require_relative "../../../support/transactional_context" + +require "test_prof/recipes/rspec/let_it_be" + +RSpec.describe "Overriding detection", :transactional do + context "when report_duplicates was set as :raise" do + context "when let_it_be redefined" do + context "when on same nested level" do + it "raises a duplication error" do + expect do + TestProf::LetItBe.configure do |config| + config.report_duplicates = :raise + end + + RSpec.describe "let_it_be on same nested level" do + include TestProf::FactoryBot::Syntax::Methods + + let_it_be(:user) { create(:user) } + let_it_be(:user) { create(:user) } + end + end.to raise_error(TestProf::LetItBe::DuplicationError) + end + end + + context "when nested level is 2" do + it "raises a duplication error" do + expect do + TestProf::LetItBe.configure do |config| + config.report_duplicates = :raise + end + + RSpec.describe "let_it_be in nested context" do + include TestProf::FactoryBot::Syntax::Methods + + let_it_be(:user) { create(:user) } + + context "nested context level 2" do + let_it_be(:user) { create(:user) } + end + end + end.to raise_error(TestProf::LetItBe::DuplicationError) + end + end + + context "when nested level is 3" do + it "raises a duplication error" do + expect do + TestProf::LetItBe.configure do |config| + config.report_duplicates = :raise + end + + RSpec.describe "let_it_be in nested context" do + include TestProf::FactoryBot::Syntax::Methods + + let_it_be(:user) { create(:user) } + + context "nested context level 2" do + context "nested context level 3" do + let_it_be(:user) { create(:user) } + end + end + end + end.to raise_error(TestProf::LetItBe::DuplicationError) + end + end + end + + context "when defined let and let_it_be" do + it "does not raise a duplication error" do + expect do + TestProf::LetItBe.configure do |config| + config.report_duplicates = :raise + end + + RSpec.describe "let_it_be and let" do + include TestProf::FactoryBot::Syntax::Methods + + let(:user) { create(:user) } + + context "nested context level 2" do + let_it_be(:user) { create(:user) } + end + end + end.not_to raise_error + end + end + end + + context "when report_duplicates was set as :warn" do + let(:warning_msg) { "let_it_be(:user) was redefined in nested group" } + + before do + allow(::RSpec).to receive(:warn_with).with(warning_msg) + end + + context "when let_it_be redefined" do + context "when on same nested level" do + it "warns a duplication message" do + RSpec.describe "let_it_be on same nested level" do + include TestProf::FactoryBot::Syntax::Methods + + TestProf::LetItBe.configure do |config| + config.report_duplicates = :warn + end + + let_it_be(:user) { create(:user) } + let_it_be(:user) { create(:user) } + end.run + + expect(::RSpec).to have_received(:warn_with).with(warning_msg).once + end + end + + context "when nested level is 2" do + it "warns a duplication message" do + RSpec.describe "let_it_be in nested context" do + include TestProf::FactoryBot::Syntax::Methods + + TestProf::LetItBe.configure do |config| + config.report_duplicates = :warn + end + + let_it_be(:user) { create(:user) } + + context "nested context" do + let_it_be(:user) { create(:user) } + end + end.run + + expect(::RSpec).to have_received(:warn_with).with(warning_msg).once + end + end + + context "when nested level is 3" do + it "warns a duplication message" do + RSpec.describe "let_it_be in nested context" do + include TestProf::FactoryBot::Syntax::Methods + + TestProf::LetItBe.configure do |config| + config.report_duplicates = :warn + end + + let_it_be(:user) { create(:user) } + + context "nested context level 2" do + context "nested context level 3" do + let_it_be(:user) { create(:user) } + end + end + end.run + + expect(::RSpec).to have_received(:warn_with).with(warning_msg).once + end + end + end + + context "when defined let and let_it_be" do + it "does not warn a duplication message" do + RSpec.describe "let_it_be and let" do + include TestProf::FactoryBot::Syntax::Methods + + TestProf::LetItBe.configure do |config| + config.report_duplicates = :raise + end + + let(:user) { create(:user) } + + context "nested context level 2" do + let_it_be(:user) { create(:user) } + end + end.run + + expect(::RSpec).not_to have_received(:warn_with) + end + end + end +end diff --git a/spec/integrations/fixtures/rspec/logging_fixture.rb b/spec/integrations/fixtures/rspec/logging_fixture.rb new file mode 100644 index 00000000..a42c6e72 --- /dev/null +++ b/spec/integrations/fixtures/rspec/logging_fixture.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require_relative "../../../support/ar_models" +require "test-prof" + +module Rails + class << self + attr_writer :logger + + def logger + @logger ||= Logger.new(IO::NULL) + end + end +end + +require "test_prof/recipes/logging" + +TestProf.configure do |config| + config.output_dir = "../../../../tmp/test_prof" +end + +describe "Logging" do + context "global", test: :global do + context "ActiveRecord" do + let(:user) { TestProf::FactoryBot.create(:user) } + + it "generates users" do + user2 = TestProf::FactoryBot.create(:user, name: "a") + Rails.logger.debug "USER: #{user2.name}" + expect(user.name).not_to eq user2.name + end + end + + context "all" do + let(:user) { TestProf::FactoryBot.create(:user) } + + it "generates users" do + user2 = TestProf::FactoryBot.create(:user, name: "b") + Rails.logger.debug "USER: #{user2.name}" + expect(user.name).not_to eq user2.name + end + end + end + + context "tags", test: :tags do + context "tags active_record", log: :ar do + let(:user) { Fabricate(:user) } + + it "generates users" do + user2 = Fabricate(:user, name: "invisible") + Rails.logger.debug "USER: #{user2.name}" + expect(user.name).not_to eq user2.name + end + end + + context "tags all", log: :all do + let(:user) { Fabricate(:user) } + + it "generates users" do + user2 = Fabricate(:user, name: "visible") + Rails.logger.debug "USER: #{user2.name}" + expect(user.name).not_to eq user2.name + end + end + end +end diff --git a/spec/integrations/fixtures/rspec/memory_prof_fixture.rb b/spec/integrations/fixtures/rspec/memory_prof_fixture.rb new file mode 100644 index 00000000..23d55bbc --- /dev/null +++ b/spec/integrations/fixtures/rspec/memory_prof_fixture.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test-prof" +require "securerandom" + +describe "Examples allocations" do + it "allocates 500 objects" do + 500.times.map { SecureRandom.hex } + end + + it "allocates 1000 objects" do + 1000.times.map { SecureRandom.hex } + end + + it "allocates 10_000 objects" do + 10_000.times.map { SecureRandom.hex } + end +end + +describe "Groups Allocations" do + context "with 500 allocations" do + before(:context) do + @array = 500.times.map { SecureRandom.hex } + end + + it { 1 } + end + + context "with 1000 allocations" do + before(:context) do + @array = 1000.times.map { SecureRandom.hex } + end + + it "does not allocate anything" do + end + end + + context "with 10_000 allocations" do + before(:context) do + @array = 10_000.times.map { SecureRandom.hex } + end + + it "does not allocate anything" do + end + end +end diff --git a/spec/integrations/fixtures/rspec/tps_prof_fixture.rb b/spec/integrations/fixtures/rspec/tps_prof_fixture.rb new file mode 100644 index 00000000..931688eb --- /dev/null +++ b/spec/integrations/fixtures/rspec/tps_prof_fixture.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require "active_support" +require "test-prof" + +describe "Something" do + it "sleeps a bit" do + sleep 0.12 + expect(true).to eq true + end + + it "sleeps a bit more" do + sleep 0.43 + expect(true).to eq true + end +end + +describe "Another something" do + before(:all) { sleep 1.3 } + + it "do nothing" do + expect(true).to eq true + end + + it "sleeps too long" do + sleep 1.2 + expect(true).to eq true + end +end diff --git a/spec/integrations/fixtures/rspec/vernier_fixture.rb b/spec/integrations/fixtures/rspec/vernier_fixture.rb new file mode 100644 index 00000000..d5d71450 --- /dev/null +++ b/spec/integrations/fixtures/rspec/vernier_fixture.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../../../../../lib", __FILE__) +require "test-prof" + +TestProf.configure do |config| + config.output_dir = "../../../../tmp/test_prof" +end + +shared_examples_for "Something" do + it "always passes", :vernier do + expect(true).to eq true + end +end + +describe "One more thing" do + include_examples "Something" +end diff --git a/spec/integrations/let_it_be_spec.rb b/spec/integrations/let_it_be_spec.rb index 4ecae187..9f522549 100644 --- a/spec/integrations/let_it_be_spec.rb +++ b/spec/integrations/let_it_be_spec.rb @@ -18,4 +18,10 @@ expect(output).to include("0 failures") end + + specify "it detects let_it_be override" do + output = run_rspec("let_it_be_nested") + + expect(output).to include("0 failures") + end end diff --git a/spec/integrations/logging_spec.rb b/spec/integrations/logging_spec.rb new file mode 100644 index 00000000..6aeaa747 --- /dev/null +++ b/spec/integrations/logging_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +describe "Logging" do + context "RSpec integration" do + specify "global all", :aggregate_failures do + output = run_rspec("logging", env: {"LOG" => "all"}, options: "--tag test:global") + + expect(output).to include("examples, 0 failures") + + expect(output).to include("INSERT INTO") + expect(output).to include("USER: a") + expect(output).to include("USER: b") + end + + specify "global active record", :aggregate_failures do + output = run_rspec("logging", env: {"LOG" => "ar"}, options: "--tag test:global") + + expect(output).to include("examples, 0 failures") + + expect(output).to include("INSERT INTO") + expect(output).not_to include("USER: a") + expect(output).not_to include("USER: b") + end + + specify "tags", :aggregate_failures do + output = run_rspec("logging", env: {"LOG" => "ar"}, options: "--tag test:tags") + + expect(output).to include("examples, 0 failures") + + expect(output).to include("INSERT INTO") + expect(output).not_to include("USER: invisible") + expect(output).to include("USER: visible") + end + end +end diff --git a/spec/integrations/memory_prof_minitest_spec.rb b/spec/integrations/memory_prof_minitest_spec.rb new file mode 100644 index 00000000..a58dbeea --- /dev/null +++ b/spec/integrations/memory_prof_minitest_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +number_regex = /\d+\.?\d{0,2}/ +memory_human_regex = /#{number_regex}[KMGTPEZ]?B/ +percent_regex = /#{number_regex}%/ + +describe "MemoryProf Minitest" do + specify "with default options", :aggregate_failures do + output = run_minitest("memory_prof", env: {"TEST_MEM_PROF" => "test"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 5 examples (by RSS):") + + expect(output).to match(/allocates 10_000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 1000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 500 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 100 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates nothing \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + end + + specify "in RSS mode", :aggregate_failures do + output = run_minitest("memory_prof", env: {"TEST_MEM_PROF" => "rss"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 5 examples (by RSS):") + + expect(output).to match(/allocates 10_000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 1000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 500 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 100 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates nothing \(\.\/memory_prof_fixture.rb:\d+\) – \+#{memory_human_regex} \(#{percent_regex}\)/) + end + + if RUBY_ENGINE != "jruby" + specify "in allocations mode", :aggregate_failures do + output = run_minitest("memory_prof", env: {"TEST_MEM_PROF" => "alloc"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Total allocations: #{number_regex}/) + + expect(output).to include("Top 5 examples (by allocations):") + + expect(output).to match(/allocates 10_000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 1000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 500 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 100 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates nothing \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + end + end + + specify "with top_count", :aggregate_failures do + output = run_minitest("memory_prof", env: {"TEST_MEM_PROF" => "rss", "TEST_MEM_PROF_COUNT" => "3"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 3 examples (by RSS):") + end +end diff --git a/spec/integrations/memory_prof_rspec_spec.rb b/spec/integrations/memory_prof_rspec_spec.rb new file mode 100644 index 00000000..2b584964 --- /dev/null +++ b/spec/integrations/memory_prof_rspec_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +number_regex = /\d+\.?\d{0,2}/ +memory_human_regex = /#{number_regex}[KMGTPEZ]?B/ +percent_regex = /#{number_regex}%/ + +describe "MemoryProf RSpec" do + specify "with default options", :aggregate_failures do + output = run_rspec("memory_prof", env: {"TEST_MEM_PROF" => "test", "TEST_MEM_PROF_COUNT" => "3"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 3 groups (by RSS):") + expect(output).to include("Top 3 examples (by RSS):") + end + + specify "in RSS mode", :aggregate_failures do + output = run_rspec("memory_prof", env: {"TEST_MEM_PROF" => "rss", "TEST_MEM_PROF_COUNT" => "3"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 3 groups (by RSS):") + expect(output).to include("Top 3 examples (by RSS):") + end + + if RUBY_ENGINE != "jruby" + specify "in allocations mode", :aggregate_failures do + output = run_rspec("memory_prof", env: {"TEST_MEM_PROF" => "alloc", "TEST_MEM_PROF_COUNT" => "3"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Total allocations: #{number_regex}/) + + expect(output).to include("Top 3 groups (by allocations):") + expect(output).to include("Top 3 examples (by allocations):") + + expect(output).to match(/with 10_000 allocations \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/with 1000 allocations \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/with 500 allocations \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + + expect(output).to match(/allocates 10_000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 1000 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + expect(output).to match(/allocates 500 objects \(\.\/memory_prof_fixture.rb:\d+\) – \+#{number_regex} \(#{percent_regex}\)/) + end + end + + specify "with top_count", :aggregate_failures do + output = run_rspec("memory_prof", env: {"TEST_MEM_PROF" => "rss", "TEST_MEM_PROF_COUNT" => "4"}) + + expect(output).to include("MemoryProf results") + expect(output).to match(/Final RSS: #{memory_human_regex}/) + + expect(output).to include("Top 4 groups (by RSS):") + expect(output).to include("Top 4 examples (by RSS):") + end +end diff --git a/spec/integrations/profilers_spec.rb b/spec/integrations/profilers_spec.rb index f4f4009c..3319b48f 100644 --- a/spec/integrations/profilers_spec.rb +++ b/spec/integrations/profilers_spec.rb @@ -4,7 +4,8 @@ begin require "stackprof" require "ruby-prof" - rescue LoadError # rubocop:disable Lint/HandleExceptions + require "vernier" + rescue LoadError end describe "general profilers", skip: !PROFILERS_AVAILABLE do @@ -24,6 +25,14 @@ expect(output).to include("RubyProf report generated") expect(output).to include("0 failures") end + + specify "examples only" do + output = run_rspec("ruby_prof", env: {"TEST_RUBY_PROF" => "1", "TEST_RUBY_PROF_BOOT" => "0"}) + + expect(output).to include("RubyProf enabled for examples") + expect(output).to include("RubyProf report generated") + expect(output).to include("0 failures") + end end context "stackprof" do @@ -39,7 +48,34 @@ expect(output).to include("StackProf (raw) enabled globally") expect(output).to include("StackProf report generated") + expect(output).to include("StackProf JSON report generated") + expect(output).to include("0 failures") + end + end + + context "vernier" do + specify "per example" do + output = run_rspec("vernier") + + expect(output).to match(/Vernier report generated.+vernier_fixture-rb-1-1/) + expect(output).to include("0 failures") + end + + specify "global" do + output = run_rspec("vernier", env: {"TEST_VERNIER" => "1"}) + + expect(output).to include("Vernier enabled globally") + expect(output).to include("Vernier report generated") + expect(output).to include("0 failures") + end + + specify "with hooks vernier contains rails events" do + output = run_rspec("vernier", env: {"TEST_VERNIER_HOOKS" => "rails"}) + sample_rails_event = "load_config_initializer.railties" + vernier_report = File.read("tmp/test_prof/vernier-report-wall--vernier_fixture-rb-1-1-.json") + expect(output).to include("0 failures") + expect(vernier_report).to match(/#{sample_rails_event}/) end end end diff --git a/spec/integrations/rubocop_spec.rb b/spec/integrations/rubocop_spec.rb index 021206b1..cb2c4037 100644 --- a/spec/integrations/rubocop_spec.rb +++ b/spec/integrations/rubocop_spec.rb @@ -6,6 +6,7 @@ output = run_rubocop("aggregate_failures", cop: "RSpec/AggregateExamples") expect(output).to include("1 offense detected") + expect(output).to include("RSpec/AggregateExamples") end end end diff --git a/spec/integrations/tag_prof_spec.rb b/spec/integrations/tag_prof_spec.rb index 2747620e..61d81a10 100644 --- a/spec/integrations/tag_prof_spec.rb +++ b/spec/integrations/tag_prof_spec.rb @@ -1,36 +1,108 @@ # frozen_string_literal: true describe "TagProf" do - specify "it works", :aggregate_failures do - output = run_rspec("tag_prof", env: {"TAG_PROF" => "type"}) - - expect(output).to include("TagProf report for type") - expect(output).to match(/type\s+time\s+total\s+%total\s+%time\s+avg\n\n/) - expect(output).to match(/fail\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) - expect(output).to match(/pass\s+\d{2}:\d{2}\.\d{3}\s+2\s+/) - expect(output).to match(/__unknown__\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) - end + context "rspec" do + specify "it works", :aggregate_failures do + output = run_rspec("tag_prof", env: {"TAG_PROF" => "type"}) + + expect(output).to include("TagProf report for type") + expect(output).to match(/type\s+time\s+total\s+%total\s+%time\s+avg\n\n/) + expect(output).to match(/fail\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) + expect(output).to match(/pass\s+\d{2}:\d{2}\.\d{3}\s+2\s+/) + expect(output).to match(/__unknown__\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) + end + + specify "html report" do + output = run_rspec("tag_prof", env: {"TAG_PROF" => "type", "TAG_PROF_FORMAT" => "html"}) - specify "html report" do - output = run_rspec("tag_prof", env: {"TAG_PROF" => "type", "TAG_PROF_FORMAT" => "html"}) + expect(output).to include("TagProf report generated:") + + expect(File.exist?("tmp/test_prof/tag-prof.html")).to eq true + end - expect(output).to include("TagProf report generated:") + context "with events" do + specify "it works", :aggregate_failures do + output = run_rspec( + "tag_prof", + env: {"TAG_PROF" => "type", "TAG_PROF_EVENT" => "test.event,test.event2"} + ) - expect(File.exist?("tmp/test_prof/tag-prof.html")).to eq true + expect(output).to include("TagProf report for type") + expect(output).to match(/type\s+time\s+test\.event\s+test.event2\s+total\s+%total\s+%time\s+avg\n\n/) + expect(output).to match(/fail\s+\d{2}:\d{2}\.\d{3}\s+00:23.000\s+00:00.000\s+1\s+/) + expect(output).to match(/pass\s+\d{2}:\d{2}\.\d{3}\s+00:12.420\s+00:14.041\s+2\s+/) + expect(output).to match(/__unknown__\s+\d{2}:\d{2}\.\d{3}\s+00:00.000\s+00:00.000\s+1\s+/) + end + end end - context "with events" do - specify "it works", :aggregate_failures do - output = run_rspec( - "tag_prof", - env: {"TAG_PROF" => "type", "TAG_PROF_EVENT" => "test.event,test.event2"} - ) + context "minitest" do + subject(:output) { run_minitest(path, env: env, chdir: chdir) } + let(:path) { "tag_prof" } + let(:env) { {"TAG_PROF" => "type"} } + let(:chdir) { nil } + it "includes tag prof report" do expect(output).to include("TagProf report for type") - expect(output).to match(/type\s+time\s+test\.event\s+test.event2\s+total\s+%total\s+%time\s+avg\n\n/) - expect(output).to match(/fail\s+\d{2}:\d{2}\.\d{3}\s+00:23.000\s+00:00.000\s+1\s+/) - expect(output).to match(/pass\s+\d{2}:\d{2}\.\d{3}\s+00:12.420\s+00:14.041\s+2\s+/) - expect(output).to match(/__unknown__\s+\d{2}:\d{2}\.\d{3}\s+00:00.000\s+00:00.000\s+1\s+/) + end + + it "includes tag prof report headers" do + expect(output).to match(/type\s+time\s+total\s+%total\s+%time\s+avg\n\n/) + end + + context "when test suite is run from test file directory" do + it "includes total time spent and number of files tested for integrations directory" do + expect(output).to match(/integrations\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) + end + end + + context "when test suite is run from app root directory" do + let(:chdir) { File.expand_path("") } + let(:path) { "spec/integrations/fixtures/minitest/tag_prof" } + + it "includes total time spent and number of files tested for integrations directory" do + expect(output).to match(/integrations\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) + end + end + + context "when test suite is run with event_prof" do + let(:env) { {"TAG_PROF" => "type", "TAG_PROF_EVENT" => "test.event"} } + it "includes event name in tag prof report headers" do + expect(output).to match(/test.event/) + end + + it "includes event time in data reported for integrations directory " do + expect(output).to match(/integrations\s+\d{2}:\d{2}\.\d{3}\s+\d{2}:\d{2}\.\d{3}\s+1\s+/) + end + end + + context "when report format is HTML" do + let(:env) { {"TAG_PROF" => "type", "TAG_PROF_FORMAT" => "html"} } + + it "generates an html report and gives its location" do + output = run_rspec("tag_prof", env: env) + + expect(output).to include("TagProf report generated:") + expect(File.exist?("tmp/test_prof/tag-prof.html")).to eq true + end + end + + context "when root test directory is not named 'test' or 'spec'" do + let(:path) { "tmp/subdirectory_not_found" } + let(:chdir) { File.expand_path("") } + + before do + test_content = File.read("spec/integrations/fixtures/minitest/tag_prof_fixture.rb") + File.write("tmp/subdirectory_not_found_fixture.rb", test_content) + end + + it "reports the statistic for the test result with an explicit error message" do + expect(output).to match(/__unknown__/) + end + + after do + File.delete("tmp/subdirectory_not_found_fixture.rb") + end end end end diff --git a/spec/integrations/tps_prof_spec.rb b/spec/integrations/tps_prof_spec.rb new file mode 100644 index 00000000..9ab0b4d7 --- /dev/null +++ b/spec/integrations/tps_prof_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +describe "TPSProf" do + context "RSpec" do + specify "with default options", :aggregate_failures do + output = run_rspec("tps_prof", env: {"TPS_PROF" => "1", "TPS_PROF_MIN" => "2"}) + + expect(output).to include("TPSProf enabled (top-10)") + expect(output).to match(/Total TPS \(tests per second\): 2\.\d+/) + + expect(output).to match(/Another something \(\.\/tps_prof_fixture\.rb:\d+\) – 0\.\d+ TPS \(00:\d{2}\.\d{3} \/ 2\), group time: 00:01\.\d{3}/) + expect(output).to match(/Something \(\.\/tps_prof_fixture\.rb:\d+\) – 3\.\d+ TPS \(00:\d{2}\.\d{3} \/ 2\), group time: 00:\d{2}\.\d{3}/) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 75dfcc41..c05000a3 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -2,8 +2,8 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) begin - require "pry-byebug" -rescue LoadError # rubocop:disable Lint/HandleExceptions + require "debug" unless ENV["CI"] +rescue LoadError end require "open3" diff --git a/spec/support/ar_models.rb b/spec/support/ar_models.rb index 3a72f0e2..7b5d640b 100644 --- a/spec/support/ar_models.rb +++ b/spec/support/ar_models.rb @@ -1,8 +1,13 @@ # frozen_string_literal: true +# Set RAILS_ENV explicitily, so configurations could be picked up +ENV["RAILS_ENV"] = "test" + +require "logger" require "active_record" require "fabrication" require "test_prof" +require "active_support/inflector" require "test_prof/factory_bot" require "activerecord-jdbc-adapter" if defined? JRUBY_VERSION @@ -13,18 +18,34 @@ rescue LoadError end +def multi_db? + return false unless ActiveRecord::Base.respond_to? :connects_to + return false unless ENV["MULTI_DB"] + + ENV["MULTI_DB"] == "true" +end + DB_CONFIG = if ENV["DB"] == "sqlite-file" FileUtils.mkdir_p TestProf.config.output_dir db_path = File.join(TestProf.config.output_dir, "testdb.sqlite") FileUtils.rm(db_path) if File.file?(db_path) {adapter: "sqlite3", database: db_path} - elsif ENV["DB"] == "postgres" + elsif ENV["DB"] == "postgres" || ENV["DB"] == "mysql" require "active_record/database_configurations" + url = ENV.fetch("DATABASE_URL") do + case ENV["DB"] + when "postgres" + ENV.fetch("POSTGRES_URL") + when "mysql" + ENV.fetch("MYSQL_URL") + end + end + config = ActiveRecord::DatabaseConfigurations::UrlConfig.new( "test", "primary", - ENV.fetch("DATABASE_URL"), + url, {"database" => ENV.fetch("DB_NAME", "test_prof_test")} ) config.respond_to?(:configuration_hash) ? config.configuration_hash : config.config @@ -32,38 +53,101 @@ {adapter: "sqlite3", database: ":memory:"} end -ActiveRecord::Base.establish_connection(**DB_CONFIG) +if multi_db? + FileUtils.mkdir_p TestProf.config.output_dir + db_comments_path = File.join(TestProf.config.output_dir, "testdb_comments.sqlite") + FileUtils.rm(db_comments_path) if File.file?(db_comments_path) + DB_CONFIG_COMMENTS = {adapter: "sqlite3", database: db_comments_path} + ActiveRecord::Base.configurations = {test: {primary: DB_CONFIG, comments: DB_CONFIG_COMMENTS}} -# #truncate_tables is not supported in older Rails, let's just ignore the failures -ActiveRecord::Base.connection.truncate_tables(*ActiveRecord::Base.connection.tables) rescue nil # rubocop:disable Style/RescueModifier + class ApplicationRecord < ActiveRecord::Base + self.abstract_class = true -ActiveRecord::Schema.define do - using_pg = ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + connects_to database: {writing: :primary, reading: :primary} + end - enable_extension "pgcrypto" if using_pg + class CommentsRecord < ApplicationRecord + self.abstract_class = true - create_table :users, id: (using_pg ? :uuid : :bigint), if_not_exists: true do |t| - t.string :name - t.string :tag + connects_to database: {writing: :comments, reading: :comments} end - create_table :posts, if_not_exists: true do |t| - t.text :text - if using_pg - t.uuid :user_id - else - t.bigint :user_id + class Comment < CommentsRecord + belongs_to :user, dependent: :destroy + end + + ApplicationRecord.establish_connection unless ENV["DRY_RUN"] == "true" + CommentsRecord.establish_connection unless ENV["DRY_RUN"] == "true" +else + ActiveRecord::Base.configurations = {test: DB_CONFIG} + class ApplicationRecord < ActiveRecord::Base + self.abstract_class = true + end + + class Comment < ApplicationRecord + belongs_to :user, dependent: :destroy + end + + ActiveRecord::Base.establish_connection(**DB_CONFIG) unless ENV["DRY_RUN"] == "true" +end + +unless ENV["DRY_RUN"] == "true" + # #truncate_tables is not supported in older Rails, let's just ignore the failures + ActiveRecord::Base.connection.truncate_tables(*ActiveRecord::Base.connection.tables) rescue nil # rubocop:disable Style/RescueModifier + + ActiveRecord::Schema.define do + using_pg = ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + + enable_extension "pgcrypto" if using_pg + + create_table :users, id: (using_pg ? :uuid : :bigint), if_not_exists: true do |t| + t.string :name + t.string :tag + end + + create_table :posts, if_not_exists: true do |t| + t.text :text + if using_pg + t.uuid :user_id + else + t.bigint :user_id + end + t.foreign_key :users + t.timestamps + end + + create_table :supercalifragilisticexpialidocious, id: (using_pg ? :uuid : :bigint), if_not_exists: true do |t| + t.string :name + t.string :tag + end + end + + ActiveRecord::Base.establish_connection DB_CONFIG_COMMENTS if multi_db? + ActiveRecord::Schema.define do + create_table :comments, if_not_exists: true do |t| + t.string :post_id # String because it could be a UUID + t.string :user_id # String because it could be a UUID + t.string :comment end - t.foreign_key :users - t.timestamps end end -ActiveRecord::Base.logger = Logger.new($stdout) if ENV["LOG"] +if multi_db? + ActiveRecord::Base.establish_connection + CommentsRecord.establish_connection DB_CONFIG_COMMENTS +end + +ActiveRecord::Base.logger = + if ENV["DEBUG"] + Logger.new($stdout) + else + Logger.new(IO::NULL) + end -class User < ActiveRecord::Base +class User < ApplicationRecord validates :name, presence: true has_many :posts, dependent: :destroy + has_many :comments, dependent: :destroy def clone copy = dup @@ -72,12 +156,22 @@ def clone end end -class Post < ActiveRecord::Base +class Post < ApplicationRecord belongs_to :user attr_accessor :dirty end +class Supercalifragilisticexpialidocious < ApplicationRecord + validates :name, presence: true + + def clone + copy = dup + copy.name = "#{name} (cloned)" + copy + end +end + TestProf::FactoryBot.define do factory :user do sequence(:name) { |n| "John #{n}" } @@ -105,6 +199,10 @@ class Post < ActiveRecord::Base user { create(:user) } end + trait :with_tagged_user do + association :user, tag: "some tag" + end + trait :with_traited_user do association :user, factory: %i[user traited] end @@ -113,13 +211,49 @@ class Post < ActiveRecord::Base association :user, factory: %i[user other_trait] end end + + factory :comment do + comment { "Interesting Post!" } + + trait :with_post do + after(:create) do + TestProf::FactoryBot.create(:post) + end + end + + trait :with_user do + after(:create) do + TestProf::FactoryBot.create(:user) + end + end + end + + factory :supercalifragilisticexpialidocious do + sequence(:name) { |n| "John #{n}" } + + trait :traited do + tag { "traited" } + end + + trait :other_trait_with_very_long_name do + tag { "other_trait_with_very_long_name" } + end + end end Fabricator(:user) do - name Fabricate.sequence(:name) { |n| "John #{n}" } + name { sequence(:name) { |n| "John #{n}" } } end Fabricator(:post) do - text Fabricate.sequence(:text) { |n| "Post ##{n}}" } + text { sequence(:text) { |n| "Post ##{n}}" } } user end + +Fabricator(:alice_post, from: :post) do + user { Fabricate(:user, name: "Alice") } +end + +Fabricator(:supercalifragilisticexpialidocious) do + name { sequence(:name) { |n| "John #{n}" } } +end diff --git a/spec/support/integration_helpers.rb b/spec/support/integration_helpers.rb index 27b57d65..30ba3a1a 100644 --- a/spec/support/integration_helpers.rb +++ b/spec/support/integration_helpers.rb @@ -11,7 +11,7 @@ module IntegrationHelpers RSPEC_STUB = File.join(__dir__, "../../bin/rspec") def run_rspec(path, chdir: nil, success: true, env: {}, options: "") - command = "#{RUBY_RUNNER} #{RSPEC_STUB} #{options} #{path}_fixture.rb" + command = "#{RUBY_RUNNER} #{RSPEC_STUB} #{options} -rlogger #{path}_fixture.rb" output, err, status = Open3.capture3( env, command, @@ -28,7 +28,7 @@ def run_rspec(path, chdir: nil, success: true, env: {}, options: "") end def run_minitest(path, chdir: nil, success: true, env: {}) - command = "#{RUBY_RUNNER} #{path}_fixture.rb #{env["TESTOPTS"]}" + command = "#{RUBY_RUNNER} -rlogger #{path}_fixture.rb #{env["TESTOPTS"]}" output, err, status = Open3.capture3( env, @@ -49,7 +49,7 @@ def run_rubocop(path, cop:) fullpath = File.join(__dir__, "../integrations/fixtures/rubocop", "#{path}_fixture.rb") test_prof_lib = File.join(__dir__, "../../lib") - command = "rubocop -r test_prof/rubocop.rb --force-default-config --only #{cop} #{fullpath} 2>&1" + command = "bundle exec rubocop --plugin test-prof --force-default-config --only #{cop} #{fullpath} 2>&1" output, err, _status = Open3.capture3( {"RUBYOPT" => "-I#{test_prof_lib}"}, diff --git a/spec/support/transactional_context.rb b/spec/support/transactional_context.rb index 8e12b86a..19b6afcc 100644 --- a/spec/support/transactional_context.rb +++ b/spec/support/transactional_context.rb @@ -5,11 +5,11 @@ shared_context "transactional", transactional: true do prepend_before(:each) do ActiveRecord::Base.connection.begin_transaction(joinable: false) - Isolator.transactions_threshold += 1 if defined?(Isolator) + ::Isolator.incr_thresholds! if defined?(::Isolator) end append_after(:each) do - Isolator.transactions_threshold -= 1 if defined?(Isolator) + ::Isolator.decr_thresholds! if defined?(::Isolator) ActiveRecord::Base.connection.rollback_transaction unless ActiveRecord::Base.connection.open_transactions.zero? end diff --git a/spec/test_prof/any_fixture_spec.rb b/spec/test_prof/any_fixture_spec.rb index b65ba97f..42ee24cf 100644 --- a/spec/test_prof/any_fixture_spec.rb +++ b/spec/test_prof/any_fixture_spec.rb @@ -61,6 +61,30 @@ expect(User.count).to eq 1 end + + context "with before_fixtures_reset callback" do + it "runs callback" do + subject.register(:user) { TestProf::FactoryBot.create(:user) } + subject.before_fixtures_reset { Post.delete_all } + TestProf::FactoryBot.create(:post) + + subject.reset + expect(User.count).to eq 0 + expect(Post.count).to eq 0 + end + end + + context "with after_fixtures_reset callback" do + it "runs callback" do + subject.register(:user) { TestProf::FactoryBot.create(:user) } + subject.after_fixtures_reset { Post.delete_all } + TestProf::FactoryBot.create(:post, user: nil) + + subject.reset + expect(User.count).to eq 0 + expect(Post.count).to eq 0 + end + end end describe "#register_dump" do @@ -69,6 +93,10 @@ User.delete_all end + before do + skip unless ActiveRecord::Base.connection.adapter_name.match?(/(sqlite|postgres)/i) + end + after do subject.reset # Reset manually data populated via CLI tools @@ -86,7 +114,7 @@ tmp_user.destroy! crypto = "crypto$5a$31$OsQLJ8tnIkCChMDcd?AiD?S.c/xUwe.Sk" - how_are_you = "How are you doing? OK?" + how_are_you = "How are you doing?\nOK?" expect do subject.register_dump("users") do @@ -284,14 +312,4 @@ expect(User.count).to eq 1 end end - - describe "#reporting_enabled" do - it "returns the config value" do - raise "Remove deprecated #reporting_enabled" if TestProf::VERSION >= "1.1" - - allow(described_class.config).to receive(:reporting_enabled) { :truth } - - expect(described_class.reporting_enabled).to eq :truth - end - end end diff --git a/spec/test_prof/before_all/adapters/active_record_spec.rb b/spec/test_prof/before_all/adapters/active_record_spec.rb new file mode 100644 index 00000000..d020823f --- /dev/null +++ b/spec/test_prof/before_all/adapters/active_record_spec.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module TestProf + module BeforeAll + def self.configure + end + end +end + +require "test_prof/before_all/adapters/active_record" + +describe TestProf::BeforeAll::Adapters::ActiveRecord do + context "when using single database", skip: (multi_db? ? "Using multiple databases" : nil) do + let(:connection_pool) { ApplicationRecord.connection_pool } + let(:connection) { connection_pool.connection } + + describe ".begin_transaction" do + subject { ::TestProf::BeforeAll::Adapters::ActiveRecord.begin_transaction } + + if ::ActiveRecord::Base.connection.pool.respond_to?(:pin_connection!) + it "calls pin_connection! on all available connections" do + expect(connection_pool).to receive(:pin_connection!).with(true) + + subject + end + else + it "calls begin_transaction on all available connections" do + expect(connection).to receive(:begin_transaction).with(a_hash_including(joinable: false)) + + subject + end + end + end + + describe ".rollback_transaction" do + subject { ::TestProf::BeforeAll::Adapters::ActiveRecord.rollback_transaction } + + if ::ActiveRecord::Base.connection.pool.respond_to?(:pin_connection!) + it "calls unpin_connection! on all available connections" do + expect(connection_pool).to receive(:unpin_connection!) + + subject + end + else + context "when not all connections have started a transaction" do + before do + # Ensure no transactions are open due to randomization of specs + connection.rollback_transaction unless connection.open_transactions.zero? + end + + it "warns when connection does not have open transaction" do + expect { subject }.to output( + /!!! before_all transaction has been already rollbacked and could work incorrectly\n/ + ).to_stderr + end + end + + context "when the connection is a transaction" do + before do + connection.begin_transaction + allow(connection).to receive(:rollback_transaction).and_call_original + end + + it "calls rollback_transaction on all available connections" do + subject + expect(connection).to have_received(:rollback_transaction) + end + end + end + end + end + + context "when using multiple databases", skip: ((!multi_db?) ? "Using single database" : nil) do + let(:connection_pool_list) { [ApplicationRecord, CommentsRecord] } + let(:connection_1) { connection_pool_list.first.connection } + let(:connection_2) { connection_pool_list.second.connection } + + describe ".begin_transaction" do + subject { ::TestProf::BeforeAll::Adapters::ActiveRecord.begin_transaction } + + it "calls begin_transaction on all available connections" do + expect(connection_1).to receive(:begin_transaction).with(a_hash_including(joinable: false)) + expect(connection_2).to receive(:begin_transaction).with(a_hash_including(joinable: false)) + + subject + end + end + + describe ".rollback_transaction" do + subject { ::TestProf::BeforeAll::Adapters::ActiveRecord.rollback_transaction } + + if ::ActiveRecord::Base.connection.pool.respond_to?(:pin_connection!) + it "calls #unpin_connection! on each connection" do + expect(connection_pool_list.first.connection_pool).to receive(:unpin_connection!) + expect(connection_pool_list.last.connection_pool).to receive(:unpin_connection!) + + subject + end + else + context "when not all connections have started a transaction" do + before do + # Ensure no transactions are open due to randomization of specs + connection_1.rollback_transaction unless connection_1.open_transactions.zero? + connection_2.rollback_transaction unless connection_2.open_transactions.zero? + connection_2.begin_transaction + end + + it "warns when connection does not have open transaction" do + expect { subject }.to output( + /!!! before_all transaction has been already rollbacked and could work incorrectly\n/ + ).to_stderr + end + end + + context "when the connection is a transaction" do + before do + connection_1.begin_transaction + connection_2.begin_transaction + end + + it "calls rollback_transaction on all available connections" do + expect(connection_1).to receive(:rollback_transaction) + expect(connection_2).to receive(:rollback_transaction) + + subject + end + end + end + end + end +end diff --git a/spec/test_prof/before_all_spec.rb b/spec/test_prof/before_all_spec.rb new file mode 100644 index 00000000..4e3c1348 --- /dev/null +++ b/spec/test_prof/before_all_spec.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +require "test_prof/before_all" + +RSpec.describe TestProf::BeforeAll do + let(:adapter) { double("Adapter") } + + before do + described_class.adapter = adapter + end + + describe ".adapter" do + context "when adapter is set" do + it "returns the adapter" do + expect(described_class.adapter).to eq(adapter) + end + end + + context "when adapter is not set" do + before do + described_class.adapter = nil + end + + let(:dry_run) { false } + + before do + allow(TestProf).to receive(:dry_run?).and_return(dry_run) + end + + it "returns the ActiveRecord adapter" do + expect(described_class.adapter).to eq(TestProf::BeforeAll::Adapters::ActiveRecord) + end + + context "when dry run mode is enabled" do + let(:dry_run) { true } + + it "returns NoopAdapter" do + expect(described_class.adapter).to eq(TestProf::BeforeAll::NoopAdapter) + end + end + end + end + + describe ".begin_transaction" do + it "calls begin_transaction on adapter" do + allow(adapter).to receive(:begin_transaction) + expect(adapter).to receive(:begin_transaction) + + described_class.begin_transaction {} + end + end + + describe ".rollback_transaction" do + it "calls rollback_transaction on adapter" do + allow(adapter).to receive(:rollback_transaction) + expect(adapter).to receive(:rollback_transaction) + + described_class.rollback_transaction {} + end + end + + describe ".setup_fixtures" do + context "when adapter supports setup_fixtures" do + it "calls setup_fixtures on adapter" do + test_object = double("TestObject") + allow(adapter).to receive(:setup_fixtures) + expect(adapter).to receive(:setup_fixtures).with(test_object) + + described_class.setup_fixtures(test_object) + end + end + + context "when adapter does not support setup_fixtures" do + it "raises ArgumentError" do + allow(adapter).to receive(:respond_to?).with(:setup_fixtures).and_return(false) + + expect { described_class.setup_fixtures(double("TestObject")) }.to raise_error(ArgumentError, "Current adapter doesn't support #setup_fixtures") + end + end + end +end + +describe TestProf::BeforeAll::Configuration do + let(:config) { described_class.new } + + describe "#before" do + it "adds a before hook to the specified type" do + block = proc {} + config.before(:begin, &block) + + expect(config.instance_variable_get(:@hooks)[:begin].before.map(&:block)).to include(block) + end + + it "raises an error for invalid hook type" do + expect { config.before(:invalid_type) {} }.to raise_error(ArgumentError, "Unknown hook type: invalid_type. Valid types: begin, rollback") + end + end + + describe "#after" do + it "adds an after hook to the specified type" do + block = proc {} + config.after(:rollback, &block) + + expect(config.instance_variable_get(:@hooks)[:rollback].after.map(&:block)).to include(block) + end + + it "raises an error for invalid hook type" do + expect { config.after(:invalid_type) {} }.to raise_error(ArgumentError, "Unknown hook type: invalid_type. Valid types: begin, rollback") + end + end + + describe "#run_hooks" do + it "runs all before and after hooks for the specified type" do + block_before = proc {} + block_after = proc {} + + config.before(:begin, &block_before) + config.after(:begin, &block_after) + + expect(block_before).to receive(:call).once + expect(block_after).to receive(:call).once + + config.run_hooks(:begin) {} + end + + it "raises an error for invalid hook type" do + expect { config.run_hooks(:invalid_type) {} }.to raise_error(ArgumentError, "Unknown hook type: invalid_type. Valid types: begin, rollback") + end + end +end diff --git a/spec/test_prof/cops/rspec/aggregate_examples/its_spec.rb b/spec/test_prof/cops/rspec/aggregate_examples/its_spec.rb index 863160d5..f9382dd3 100644 --- a/spec/test_prof/cops/rspec/aggregate_examples/its_spec.rb +++ b/spec/test_prof/cops/rspec/aggregate_examples/its_spec.rb @@ -1,10 +1,18 @@ # frozen_string_literal: true require "cop_helper" -require "test_prof/cops/rspec/aggregate_examples" +require "rubocop/test_prof/cops/rspec/aggregate_examples" -RSpec.describe RuboCop::Cop::RSpec::AggregateExamples, ".its" do - subject(:cop) { described_class.new } +RSpec.describe RuboCop::Cop::RSpec::AggregateExamples, ".its", :config do + let(:all_cops_config) do + {"DisplayCopNames" => false} + end + + let(:cop_config) do + {"AddAggregateFailuresMetadata" => false} + end + + subject(:cop) { described_class.new(config) } # Regular `its` call with an attribute/method name, or a chain of methods # expressed as a string with dots. diff --git a/spec/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects_spec.rb b/spec/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects_spec.rb index ecf36032..e9cba2d7 100644 --- a/spec/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects_spec.rb +++ b/spec/test_prof/cops/rspec/aggregate_examples/matchers_with_side_effects_spec.rb @@ -1,10 +1,14 @@ # frozen_string_literal: true require "cop_helper" -require "test_prof/cops/rspec/aggregate_examples" +require "rubocop/test_prof/cops/rspec/aggregate_examples" RSpec.describe RuboCop::Cop::RSpec::AggregateExamples, ".matchers_with_side_effects", :config do + let(:all_cops_config) do + {"DisplayCopNames" => false} + end + subject(:cop) { described_class.new(config) } context "without side effect matchers defined in configuration" do diff --git a/spec/test_prof/cops/rspec/aggregate_examples_spec.rb b/spec/test_prof/cops/rspec/aggregate_examples_spec.rb index 357eae96..4befd970 100644 --- a/spec/test_prof/cops/rspec/aggregate_examples_spec.rb +++ b/spec/test_prof/cops/rspec/aggregate_examples_spec.rb @@ -1,9 +1,13 @@ # frozen_string_literal: true require "cop_helper" -require "test_prof/cops/rspec/aggregate_examples" +require "rubocop/test_prof/cops/rspec/aggregate_examples" RSpec.describe RuboCop::Cop::RSpec::AggregateExamples, :config do + let(:all_cops_config) do + {"DisplayCopNames" => false} + end + subject(:cop) { described_class.new(config) } let(:cop_config) do diff --git a/spec/test_prof/ext/active_record_refind_spec.rb b/spec/test_prof/ext/active_record_refind_spec.rb index cf29ba55..80931868 100644 --- a/spec/test_prof/ext/active_record_refind_spec.rb +++ b/spec/test_prof/ext/active_record_refind_spec.rb @@ -17,5 +17,17 @@ expect(ruser).not_to be_equal user expect(ruser.posts.first.text).to eq "clean" end + + it "ignores default scopes" do + scoped_post_class = Class.new(Post) do + default_scope { where.not(text: "dirty") } + end + + post = scoped_post_class.create!(text: "dirty") + + rpost = post.refind + + expect(rpost.text).to eq "dirty" + end end end diff --git a/spec/test_prof/factory_default_spec.rb b/spec/test_prof/factory_default_spec.rb new file mode 100644 index 00000000..cd1e3880 --- /dev/null +++ b/spec/test_prof/factory_default_spec.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +# Init FactoryDefault +require "test_prof/factory_default" + +TestProf::FactoryDefault.init +TestProf::FactoryDefault.disable! + +describe TestProf::FactoryDefault, :transactional do + let(:preserve_traits) { false } + let(:preserve_attributes) { false } + + before do + described_class.enable! + described_class.preserve_traits = preserve_traits + described_class.preserve_attributes = preserve_attributes + end + + after do + described_class.reset + described_class.disable! + end + + let!(:user) { TestProf::FactoryBot.create_default(:user) } + + it "re-uses the same default record" do + post = TestProf::FactoryBot.create(:post) + + expect(TestProf::FactoryBot.get_factory_default(:user)).to eq user + expect(post.user).to eq user + end + + it "re-uses default record independently of traits" do + post = TestProf::FactoryBot.create(:post, :with_traited_user) + + expect(TestProf::FactoryBot.get_factory_default(:user)).to eq user + expect(post.user).to eq user + expect(TestProf::FactoryBot.get_factory_default(:user, :traited)).to eq user + end + + it "re-uses default record independently of attributes" do + post = TestProf::FactoryBot.create(:post, :with_tagged_user) + + expect(TestProf::FactoryBot.get_factory_default(:user)).to eq user + expect(post.user).to eq user + end + + specify ".disable!(&block)" do + post = TestProf::FactoryBot.skip_factory_default { TestProf::FactoryBot.create(:post) } + + # get_factory_default should ignore disabled + expect(TestProf::FactoryBot.get_factory_default(:user)).to eq user + expect(post.user).not_to eq(user) + end + + context "when preserve_traits = true" do + let(:preserve_traits) { true } + + it "ignores default when trait is specified" do + post = TestProf::FactoryBot.create_default(:post) + post_traited = TestProf::FactoryBot.create(:post, :with_traited_user) + + expect(TestProf::FactoryBot.get_factory_default(:post).user).to eq user + expect(post.user).to eq user + + expect(TestProf::FactoryBot.get_factory_default(:post, :with_traited_user)).to eq nil + expect(post_traited.user).not_to eq user + end + + context "when has default with the trait" do + let!(:traited_user) do + user = TestProf::FactoryBot.create(:user, :traited) + TestProf::FactoryBot.set_factory_default(:user, :traited, user) + end + + it "re-uses default record for this trait" do + post = TestProf::FactoryBot.create_default(:post) + post_traited = TestProf::FactoryBot.create(:post, :with_traited_user) + + expect(TestProf::FactoryBot.get_factory_default(:post).user).to eq user + expect(post.user).to eq user + expect(TestProf::FactoryBot.get_factory_default(:user, :traited)).to eq traited_user + expect(post_traited.user).to eq traited_user + end + end + end + + context "when preserve_attributes = true" do + let(:preserve_attributes) { true } + + it "ignores default when explicit attributes don't match" do + post = TestProf::FactoryBot.create(:post, :with_tagged_user) + + expect(TestProf::FactoryBot.get_factory_default(:user, tag: "some tag")).to eq nil + expect(post.user).not_to eq user + end + + it "re-uses default when attributes match" do + user.update!(tag: "some tag") + + post = TestProf::FactoryBot.create(:post, :with_tagged_user) + + expect(TestProf::FactoryBot.get_factory_default(:user, tag: "some tag")).to eq user + expect(post.user).to eq user + end + end +end diff --git a/spec/test_prof/factory_doctor_spec.rb b/spec/test_prof/factory_doctor_spec.rb index 8f9631aa..d0f700e1 100644 --- a/spec/test_prof/factory_doctor_spec.rb +++ b/spec/test_prof/factory_doctor_spec.rb @@ -17,6 +17,16 @@ described_class.config.threshold = was_threshold end + let(:debug_queries) { @debug_queries } + around do |ex| + @debug_queries = [] + subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |_, _, _, _, details| + @debug_queries << details[:sql] + end + ex.run + ActiveSupport::Notifications.unsubscribe(subscriber) + end + describe "#result" do subject(:result) { described_class.result } @@ -26,7 +36,7 @@ expect(result).not_to be_bad expect(result.count).to eq 0 expect(result.time).to eq 0 - expect(result.queries_count).to eq 1 + expect(result.queries_count).to eq(1), debug_queries.join("\n") end it "detects one useless object" do @@ -42,7 +52,7 @@ expect(result).not_to be_bad expect(result.count).to eq 1 - expect(result.queries_count).to eq 1 + expect(result.queries_count).to eq(1), debug_queries.join("\n") expect(result.time).to be > 0 end @@ -52,7 +62,7 @@ expect(result).not_to be_bad expect(result.count).to eq 1 - expect(result.queries_count).to eq 1 + expect(result.queries_count).to eq(1), debug_queries.join("\n") expect(result.time).to be > 0 end diff --git a/spec/test_prof/factory_prof/printers/flamegraph_spec.rb b/spec/test_prof/factory_prof/printers/flamegraph_spec.rb index f5c875a2..faf5550a 100644 --- a/spec/test_prof/factory_prof/printers/flamegraph_spec.rb +++ b/spec/test_prof/factory_prof/printers/flamegraph_spec.rb @@ -3,7 +3,6 @@ describe TestProf::FactoryProf::Printers::Flamegraph do subject { described_class } - # rubocop:disable Style/BracesAroundHashParameters describe ".convert_stacks" do it "converts stacks to hierarchy Hash" do stacks = [] diff --git a/spec/test_prof/factory_prof/printers/json_spec.rb b/spec/test_prof/factory_prof/printers/json_spec.rb new file mode 100644 index 00000000..5bd8deb9 --- /dev/null +++ b/spec/test_prof/factory_prof/printers/json_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +describe TestProf::FactoryProf::Printers::Json do + let(:stacks) do + stacks = [] + stacks << %i[user account] + stacks << %i[user account] + stacks << %i[post account user account] + stacks << %i[comment post] + stacks << %i[comment user account] + stacks << [:user] + stacks << [:account] + stacks + end + + let(:stats) do + { + user: {name: :user, total_count: 5, total_time: 1.0, top_level_count: 5, top_level_time: 0.1, variations: []}, + account: {name: :account, total_count: 6, total_time: 2.0, top_level_count: 6, top_level_time: 0.2, variations: []}, + comment: {name: :comment, total_count: 2, total_time: 3.0, top_level_count: 2, top_level_time: 0.3, variations: []}, + name: {name: :post, total_count: 2, total_time: 4.0, top_level_count: 0, top_level_time: 0.4, variations: []} + } + end + + let(:result) { TestProf::FactoryProf::Result.new(stacks, stats) } + + describe "#dump" do + before do + allow(File).to receive(:write) + allow(TestProf).to receive(:artifact_path).and_return("test-prof.result.json") + end + + it "write json" do + described_class.dump(result, start_time: 0) + outpath = TestProf.artifact_path("test-prof.result.json") + expect(File).to have_received(:write).with(outpath, String).once + end + end + + describe "#convert_stats" do + before do + allow(TestProf).to receive(:now).and_return(2.0) + end + + it "calculates factories usage" do + stats = described_class.convert_stats(result, 0.0) + + expect(stats).to include({ + total_count: 15, + total_top_level_count: 13, + total_time: "00:01.000", + total_run_time: "00:02.000", + total_uniq_factories: 4 + }) + end + end +end diff --git a/spec/test_prof/factory_prof_spec.rb b/spec/test_prof/factory_prof_spec.rb index 4ced127a..477c1c3a 100644 --- a/spec/test_prof/factory_prof_spec.rb +++ b/spec/test_prof/factory_prof_spec.rb @@ -5,12 +5,15 @@ TestProf::FactoryProf.configure do |config| # turn on stacks collection config.mode = :flamegraph + config.include_variations = false end describe TestProf::FactoryProf, :transactional do before { described_class.start } after { described_class.stop } + after { TestProf::FactoryProf.config.include_variations = false } + # Ensure meta-queries have been performed before(:all) { User.first } @@ -18,10 +21,38 @@ def without_time(xs) xs.map do |x| expect(x.delete(:total_time)).to be_a(Float) expect(x.delete(:top_level_time)).to be_a(Float) + x[:variations] = without_time(x[:variations]) unless x[:variations].nil? x end end + describe "#print" do + let(:started_at) { Time.now } + + subject(:print) { described_class.print(started_at) } + + it "calls the default printer" do + expect(TestProf::FactoryProf::Printers::Simple).to receive(:dump) + + print + end + + context "when the printer is customized" do + let(:custom_printer) { double(dump: lambda { |result, start_time: nil| }) } + + before do + described_class.configure do |config| + config.printer = custom_printer + end + end + + it "calls the customer printer" do + expect(custom_printer).to receive(:dump) + print + end + end + end + describe "#result" do subject(:result) { described_class.result } @@ -47,13 +78,15 @@ def without_time(xs) ) expect(without_time(result.stats)).to eq( [ - {name: :post, total_count: 1, top_level_count: 1}, - {name: :user, total_count: 1, top_level_count: 0} + {name: :post, total_count: 1, top_level_count: 1, variations: []}, + {name: :user, total_count: 1, top_level_count: 0, variations: []} ] ) end - it "contains many stacks" do + it "contains many stacks with variations" do + TestProf::FactoryProf.config.include_variations = true + TestProf::FactoryBot.create_pair(:user) TestProf::FactoryBot.create(:post) TestProf::FactoryBot.create(:user, :with_posts) @@ -68,8 +101,13 @@ def without_time(xs) ) expect(without_time(result.stats)).to eq( [ - {name: :user, total_count: 6, top_level_count: 3}, - {name: :post, total_count: 3, top_level_count: 1} + {name: :user, total_count: 6, top_level_count: 3, variations: [ + {name: "-", top_level_count: 2, total_count: 5}, + {name: :".with_posts", top_level_count: 1, total_count: 1} + ]}, + {name: :post, total_count: 3, top_level_count: 1, variations: [ + {name: "-", top_level_count: 1, total_count: 3} + ]} ] ) end @@ -89,9 +127,11 @@ def without_time(xs) expect(result.stacks.first).to eq([:user]) end - it "contains many stacks" do + it "contains many stacks with variations" do + TestProf::FactoryProf.config.include_variations = true + Fabricate.times(2, :user) - Fabricate.create(:post) + Fabricate.create(:post, text: "some text") Fabricate.create(:user) { Fabricate.times(2, :post) } expect(result.stacks.size).to eq 4 @@ -104,8 +144,11 @@ def without_time(xs) ) expect(without_time(result.stats)).to eq( [ - {name: :user, total_count: 6, top_level_count: 3}, - {name: :post, total_count: 3, top_level_count: 1} + {name: :user, total_count: 6, top_level_count: 3, variations: [{name: "-", top_level_count: 3, total_count: 6}]}, + {name: :post, total_count: 3, top_level_count: 1, variations: [ + {name: "-", top_level_count: 0, total_count: 2}, + {name: :"[text]", top_level_count: 1, total_count: 1} + ]} ] ) end diff --git a/spec/test_prof/memory_prof/printer/number_to_human_spec.rb b/spec/test_prof/memory_prof/printer/number_to_human_spec.rb new file mode 100644 index 00000000..3e9ba739 --- /dev/null +++ b/spec/test_prof/memory_prof/printer/number_to_human_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +describe TestProf::MemoryProf::Printer::NumberToHuman do + subject { described_class } + + describe "#convert" do + let(:convert) { subject.convert(number) } + + context "when number is 0" do + let(:number) { 0 } + + it "returns 0B" do + expect(convert).to eq("0B") + end + end + + context "when number < 1KB" do + let(:number) { 700 } + + it "returns the value in bytes" do + expect(convert).to eq("700B") + end + end + + context "when number > 1024 ZB" do + let(:number) { 7 * 2**80 } + + it "returns the value in ZB" do + expect(convert).to eq("7168ZB") + end + end + + context "when number can be converted to an integer" do + let(:number) { 7 * 2**20 } + + it "returns the value as an integer" do + expect(convert).to eq("7MB") + end + end + + context "when number can not be converted to an integer" do + let(:number) { 7 * 2**20 + 550 * 2**10 } + + it "rounds the value to two digits" do + expect(convert).to eq("7.54MB") + end + end + end +end diff --git a/spec/test_prof/memory_prof/printer_spec.rb b/spec/test_prof/memory_prof/printer_spec.rb new file mode 100644 index 00000000..adb90c0e --- /dev/null +++ b/spec/test_prof/memory_prof/printer_spec.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +shared_examples "TestProf::MemoryProf::Printer" do + subject { described_class.new(tracker) } + + let(:tracker) do + instance_double( + TestProf::MemoryProf::Tracker, + top_count: 3, + total_memory: 500, + groups: groups, + examples: examples + ) + end + + let(:groups) do + [ + {name: "AnswersController", location: "./spec/controllers/answers_controller_spec.rb:3", memory: 200}, + {name: "#publish", location: "./spec/nodels/question_spec.rb:179", memory: 100}, + {name: "when email and name are present", location: "./spec/lib/import_spec.rb:34", memory: 50} + ] + end + + let(:examples) do + [ + {name: "returns nil", location: "./spec/models/reports_spec.rb:57", memory: 75}, + {name: "searches users by email", location: "./spec/searches/users_search_spec.rb:15", memory: 50}, + {name: "calculates the average number of answers", location: "./spec/lib/stats_spec.rb:44", memory: 25} + ] + end + + before do + allow(subject).to receive(:log) + end +end + +describe TestProf::MemoryProf::AllocPrinter do + include_examples "TestProf::MemoryProf::Printer" + + describe "#print" do + let(:print) { subject.print } + + context "and there are both groups and examples" do + let(:message) do + <<~MESSAGE + MemoryProf results + + Total allocations: 500 + + Top 3 groups (by allocations): + + AnswersController (./spec/controllers/answers_controller_spec.rb:3) – +200 (40.0%) + #publish (./spec/nodels/question_spec.rb:179) – +100 (20.0%) + when email an...me are present (./spec/lib/import_spec.rb:34) – +50 (10.0%) + + Top 3 examples (by allocations): + + returns nil (./spec/models/reports_spec.rb:57) – +75 (15.0%) + searches users by email (./spec/searches/users_search_spec.rb:15) – +50 (10.0%) + calculates th...ber of answers (./spec/lib/stats_spec.rb:44) – +25 (5.0%) + + MESSAGE + end + + it "prints results for groups and examples" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no examples" do + let(:examples) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Total allocations: 500 + + Top 3 groups (by allocations): + + AnswersController (./spec/controllers/answers_controller_spec.rb:3) – +200 (40.0%) + #publish (./spec/nodels/question_spec.rb:179) – +100 (20.0%) + when email an...me are present (./spec/lib/import_spec.rb:34) – +50 (10.0%) + + MESSAGE + } + + it "prints results for groups only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no groups" do + let(:groups) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Total allocations: 500 + + Top 3 examples (by allocations): + + returns nil (./spec/models/reports_spec.rb:57) – +75 (15.0%) + searches users by email (./spec/searches/users_search_spec.rb:15) – +50 (10.0%) + calculates th...ber of answers (./spec/lib/stats_spec.rb:44) – +25 (5.0%) + + MESSAGE + } + + it "prints results for examples only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no groups or examples" do + let(:groups) { [] } + let(:examples) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Total allocations: 500 + + MESSAGE + } + + it "prints results for examples only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + end +end + +describe TestProf::MemoryProf::RssPrinter do + include_examples "TestProf::MemoryProf::Printer" + + describe "#print" do + let(:print) { subject.print } + + context "and there are both groups and examples" do + let(:message) do + <<~MESSAGE + MemoryProf results + + Final RSS: 500B + + Top 3 groups (by RSS): + + AnswersController (./spec/controllers/answers_controller_spec.rb:3) – +200B (40.0%) + #publish (./spec/nodels/question_spec.rb:179) – +100B (20.0%) + when email an...me are present (./spec/lib/import_spec.rb:34) – +50B (10.0%) + + Top 3 examples (by RSS): + + returns nil (./spec/models/reports_spec.rb:57) – +75B (15.0%) + searches users by email (./spec/searches/users_search_spec.rb:15) – +50B (10.0%) + calculates th...ber of answers (./spec/lib/stats_spec.rb:44) – +25B (5.0%) + + MESSAGE + end + + it "prints results for groups and examples" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no examples" do + let(:examples) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Final RSS: 500B + + Top 3 groups (by RSS): + + AnswersController (./spec/controllers/answers_controller_spec.rb:3) – +200B (40.0%) + #publish (./spec/nodels/question_spec.rb:179) – +100B (20.0%) + when email an...me are present (./spec/lib/import_spec.rb:34) – +50B (10.0%) + + MESSAGE + } + + it "prints results for groups only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no groups" do + let(:groups) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Final RSS: 500B + + Top 3 examples (by RSS): + + returns nil (./spec/models/reports_spec.rb:57) – +75B (15.0%) + searches users by email (./spec/searches/users_search_spec.rb:15) – +50B (10.0%) + calculates th...ber of answers (./spec/lib/stats_spec.rb:44) – +25B (5.0%) + + MESSAGE + } + + it "prints results for examples only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + + context "and there are no groups or examples" do + let(:groups) { [] } + let(:examples) { [] } + + let(:message) { + <<~MESSAGE + MemoryProf results + + Final RSS: 500B + + MESSAGE + } + + it "prints results for examples only" do + print + + expect(subject).to have_received(:log).with(:info, message) + end + end + end +end diff --git a/spec/test_prof/memory_prof/tracker/rss_tool_spec.rb b/spec/test_prof/memory_prof/tracker/rss_tool_spec.rb new file mode 100644 index 00000000..eb826de1 --- /dev/null +++ b/spec/test_prof/memory_prof/tracker/rss_tool_spec.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +describe TestProf::MemoryProf::Tracker::RssTool do + subject { described_class } + + describe ".tool" do + before do + allow(subject).to receive(:os_type).and_return(os_type) + end + + context "when an OS is supported" do + let(:os_type) { :macosx } + + it "returns an rss tool" do + expect(subject.tool).to be_kind_of(subject::PS) + end + end + + context "when an OS is not supported" do + let(:os_type) { :invalid } + + it "returns nil" do + expect(subject.tool).to eq(nil) + end + end + end + + describe ".os_type" do + before do + @original_os = RbConfig::CONFIG["host_os"] + RbConfig::CONFIG["host_os"] = host_os + end + + after do + RbConfig::CONFIG["host_os"] = @original_os + end + + context "when the host OS is Linux" do + let(:host_os) { "linux" } + + it "returns :linux" do + expect(subject.os_type).to eq(:linux) + end + end + + context "when the host OS is macOS" do + let(:host_os) { "darwin22" } + + it "returns :macosx" do + expect(subject.os_type).to eq(:macosx) + end + end + + context "when the host OS is macOS" do + let(:host_os) { "mac os 14" } + + it "returns :macosx" do + expect(subject.os_type).to eq(:macosx) + end + end + + %w[solaris bsd].each do |os| + context "when the host OS is #{os}" do + let(:host_os) { "#{os} 17" } + + it "returns :unix" do + expect(subject.os_type).to eq(:unix) + end + end + end + + context "when the host OS is Windows" do + let(:host_os) { "mswin" } + + it "returns :windows" do + expect(subject.os_type).to eq(:windows) + end + end + end +end + +describe TestProf::MemoryProf::Tracker::RssTool::ProcFS do + subject { described_class.new } + + describe "#track" do + before do + io = instance_double(IO, seek: nil, gets: "46441 196384 1804 1 0 24133 0\n") + + allow(File).to receive(:open).and_return(io) + subject.instance_variable_set("@page_size", 1024) + end + + it "retrieves rss via proc statm" do + subject.track + + expect(File).to have_received(:open).with(/\/proc\/\d+\/statm/, "r") + end + + it "returns the current rss" do + expect(subject.track).to eq(201097216) + end + end +end + +describe TestProf::MemoryProf::Tracker::RssTool::PS do + subject { described_class.new } + + describe "#track" do + before do + allow(subject).to receive(:`).and_return(" RSS\n196384") + end + + it "retrieves rss via ps" do + subject.track + + expect(subject).to have_received(:`).with(/ps -o rss -p \d+/) + end + + it "returns the current rss" do + expect(subject.track).to eq(201097216) + end + end +end + +describe TestProf::MemoryProf::Tracker::RssTool::GetProcess do + subject { described_class.new } + + describe "#track" do + before do + allow(subject).to receive(:`).and_return("\n WS\n --\n201097216\n\n\n") + end + + it "retrieves rss via Get-Process" do + subject.track + + expect(subject).to have_received(:`).with(/powershell -Command "Get-Process -Id \d+ | select WS"/) + end + + it "returns the current rss" do + expect(subject.track).to eq(201097216) + end + end +end diff --git a/spec/test_prof/memory_prof/tracker_spec.rb b/spec/test_prof/memory_prof/tracker_spec.rb new file mode 100644 index 00000000..ef9340ea --- /dev/null +++ b/spec/test_prof/memory_prof/tracker_spec.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +shared_examples "TestProf::MemoryProf::Tracker" do + let(:list) { TestProf::MemoryProf::Tracker::LinkedList.new(100) } + let(:example) { double("example") } + let(:group) { double("group") } + + describe "#start" do + it "initializes a linked list" do + subject.start + + expect(subject.list).to be_kind_of(TestProf::MemoryProf::Tracker::LinkedList) + end + end + + describe "#finish" do + before do + allow(subject).to receive(:track).and_return(200) + allow(subject).to receive(:list).and_return(list) + end + + it "sets total_memory" do + subject.finish + + expect(subject.total_memory).to eq(100) + end + end + + describe "#example_started" do + let(:example_started) { subject.example_started(example, {name: :example}) } + + before do + allow(subject).to receive(:track).and_return(200) + allow(subject).to receive(:list).and_return(list) + end + + it "tracks memory at the start of an example" do + example_started + + expect(subject.list.head).to have_attributes(id: example, item: {name: :example}, memory_at_start: 200) + end + end + + describe "#example_finished" do + let(:example_finished) { subject.example_finished(example) } + + before do + list.add_node(example, {name: :example}, 200) + + allow(subject).to receive(:track).and_return(350) + allow(subject).to receive(:list).and_return(list) + end + + context "when the example memory > the memory of the top examples" do + before do + [75, 100, 125, 175, 200].each do |memory| + subject.examples << {memory: memory} + end + end + + it "adds the example to the top examples" do + example_finished + + expect(subject.examples).to match_array([ + {memory: 200}, + {memory: 175}, + {name: :example, memory: 150}, + {memory: 125}, + {memory: 100} + ]) + end + end + + context "when the example memory <= the memory of the top examples" do + before do + [175, 200, 225, 250, 275].each do |memory| + subject.examples << {memory: memory} + end + end + + it "adds the example to the top examples" do + example_finished + + expect(subject.examples).to match_array([ + {memory: 275}, + {memory: 250}, + {memory: 225}, + {memory: 200}, + {memory: 175} + ]) + end + end + end + + describe "#group_started" do + let(:group_started) { subject.group_started(group, {name: :group}) } + + before do + allow(subject).to receive(:track).and_return(200) + allow(subject).to receive(:list).and_return(list) + end + + it "tracks memory at the start of a group" do + group_started + + expect(subject.list.head).to have_attributes(id: group, item: {name: :group}, memory_at_start: 200) + end + end + + describe "#group_finished" do + let(:group_finished) { subject.group_finished(group) } + + before do + list.add_node(group, {name: :group}, 200) + + allow(subject).to receive(:track).and_return(350) + allow(subject).to receive(:list).and_return(list) + end + + context "when the group memory > the memory of the top groups" do + before do + [75, 100, 125, 175, 200].each do |memory| + subject.groups << {memory: memory} + end + end + + it "adds the group to the top groups" do + group_finished + + expect(subject.groups).to match_array([ + {memory: 200}, + {memory: 175}, + {name: :group, memory: 150}, + {memory: 125}, + {memory: 100} + ]) + end + end + + context "when the group memory <= the memory of the top groups" do + before do + [175, 200, 225, 250, 275].each do |memory| + subject.groups << {memory: memory} + end + end + + it "adds the group to the top groups" do + group_finished + + expect(subject.groups).to match_array([ + {memory: 275}, + {memory: 250}, + {memory: 225}, + {memory: 200}, + {memory: 175} + ]) + end + end + end +end + +describe TestProf::MemoryProf::AllocTracker do + subject { described_class.new(5) } + + if RUBY_ENGINE == "jruby" + it "raises an error" do + expect { subject }.to raise_error("Your Ruby Engine or OS is not supported") + end + else + it_behaves_like "TestProf::MemoryProf::Tracker" + + describe "#track" do + before do + allow(GC).to receive(:stat).and_return({total_allocated_objects: 100}) + end + + it "returns the current number of allocations" do + expect(subject.track).to eq(100) + end + end + + describe "#supported?" do + it "returns true" do + expect(subject.supported?).to be_truthy + end + end + end +end + +describe TestProf::MemoryProf::RssTracker do + subject { described_class.new(5) } + + let(:tool) { instance_double(TestProf::MemoryProf::Tracker::RssTool::PS, track: 100) } + + before do + allow(TestProf::MemoryProf::Tracker::RssTool).to receive(:tool).and_return(tool) + end + + it_behaves_like "TestProf::MemoryProf::Tracker" + + describe "#track" do + it "returns the current rss" do + expect(subject.track).to eq(100) + end + end + + describe "#supported?" do + context "when the host OS is supported" do + it "returns true" do + expect(subject.supported?).to be_truthy + end + end + + context "when the host OS is not supported" do + let(:tool) { nil } + + it "raises an error" do + expect { subject }.to raise_error("Your Ruby Engine or OS is not supported") + end + end + end +end diff --git a/spec/test_prof/memory_prof_spec.rb b/spec/test_prof/memory_prof_spec.rb new file mode 100644 index 00000000..7272199b --- /dev/null +++ b/spec/test_prof/memory_prof_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +describe TestProf::MemoryProf do + subject { described_class } + + describe ".config" do + let(:config) { subject.config } + + it "returns an instance of TestProf::MemoryProf::Configuration" do + expect(config).to be_kind_of(TestProf::MemoryProf::Configuration) + end + + describe "#mode=" do + let(:set_mode) { config.mode = value } + + context "when value is alloc" do + let(:value) { "alloc" } + + it "sets mode to alloc" do + set_mode + + expect(config.mode).to eq(:alloc) + end + end + + context "when value is rss" do + let(:value) { "rss" } + + it "sets mode to rss" do + set_mode + + expect(config.mode).to eq(:rss) + end + end + + context "when value is neither alloc or rss" do + let(:value) { "invalid" } + + it "sets mode to alloc" do + set_mode + + expect(config.mode).to eq(:rss) + end + end + end + + describe "#top_count=" do + let(:set_top_count) { config.top_count = value } + + context "when value is an integer" do + let(:value) { 5 } + + it "sets top_count to the value" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + + context "when value is a float" do + let(:value) { 5.7 } + + it "transforms the value to an integer" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + + context "when value is 0" do + let(:value) { 0 } + + it "sets top_count to 5" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + + context "when value is negative" do + let(:value) { -7 } + + it "sets top_count to 5" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + + context "when value is nil" do + let(:value) { nil } + + it "sets top_count to 5" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + + context "when value is a text" do + let(:value) { "invalid" } + + it "sets top_count to 5" do + set_top_count + + expect(config.top_count).to eq(5) + end + end + end + end + + describe ".tracker" do + let(:tracker) { subject.tracker } + + context "when mode is alloc" do + before { described_class.config.mode = "alloc" } + + if RUBY_ENGINE == "jruby" + it "raises an error" do + expect { tracker }.to raise_error("Your Ruby Engine or OS is not supported") + end + else + it "returns an instance of AllocTracker" do + expect(tracker).to be_kind_of(TestProf::MemoryProf::AllocTracker) + end + + it "sets tracker.top_count to config.top_count" do + expect(tracker.top_count).to eq(5) + end + end + end + + context "when mode is rss" do + before { described_class.config.mode = "rss" } + + it "returns an instance of RssTracker" do + expect(tracker).to be_kind_of(TestProf::MemoryProf::RssTracker) + end + + it "sets tracker.top_count to config.top_count" do + expect(tracker.top_count).to eq(5) + end + end + end + + describe ".printer" do + let(:printer) { subject.printer(tracker) } + let(:tracker) { subject.tracker } + + context "when mode is alloc" do + before { described_class.config.mode = "alloc" } + + if RUBY_ENGINE == "jruby" + it "raises an error" do + expect { printer }.to raise_error("Your Ruby Engine or OS is not supported") + end + else + it "returns an instance of AllocPrinter" do + expect(printer).to be_kind_of(TestProf::MemoryProf::AllocPrinter) + end + + it "sets printer.tracker" do + expect(printer.send(:tracker)).to eq(tracker) + end + end + end + + context "when mode is rss" do + before { described_class.config.mode = "rss" } + + it "returns an instance of RssPrinter" do + expect(printer).to be_kind_of(TestProf::MemoryProf::RssPrinter) + end + + it "sets printer.tracker" do + expect(printer.send(:tracker)).to eq(tracker) + end + end + end +end diff --git a/spec/test_prof/rspec_stamp_spec.rb b/spec/test_prof/rspec_stamp_spec.rb index eb8da850..a1d821f3 100644 --- a/spec/test_prof/rspec_stamp_spec.rb +++ b/spec/test_prof/rspec_stamp_spec.rb @@ -32,7 +32,7 @@ end describe ".apply_tags" do - let(:code) { source.split("\n") } + let(:code) { source.split("\n").map { _1 + "\n" } } let(:lines) { [1] } @@ -58,7 +58,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end context "with several examples" do @@ -98,7 +98,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end context "patch all" do @@ -122,7 +122,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end end @@ -146,7 +146,30 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected + end + end + + context "with escaped single quotes" do + let(:source) do + <<~CODE + it 'has a \\' in the description' do # with comment + expect(subject.body).to eq("Not Found") + end + CODE + end + + let(:expected) do + <<~CODE + it "has a \\' in the description", :todo do # with comment + expect(subject.body).to eq("Not Found") + end + CODE + end + + specify do + is_expected.to eq 0 + expect(code.join).to eq expected end end @@ -165,7 +188,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end @@ -184,7 +207,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end @@ -207,7 +230,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end @@ -232,7 +255,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end context "with existing tags" do @@ -254,7 +277,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end @@ -279,7 +302,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end end @@ -298,7 +321,7 @@ specify do is_expected.to eq 1 - expect(code.join("\n")).to eq source.strip + expect(code.join).to eq source end end @@ -343,7 +366,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end context "with existing tags" do @@ -373,7 +396,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end @@ -402,7 +425,7 @@ specify do is_expected.to eq 0 - expect(code.join("\n")).to eq expected.strip + expect(code.join).to eq expected end end end diff --git a/spec/test_prof/ruby_prof_spec.rb b/spec/test_prof/ruby_prof_spec.rb index db5755c2..6c81a6f8 100644 --- a/spec/test_prof/ruby_prof_spec.rb +++ b/spec/test_prof/ruby_prof_spec.rb @@ -1,6 +1,12 @@ # frozen_string_literal: true describe TestProf::RubyProf do + before do + next if defined?(::RubyProf::WALL_TIME) + + stub_const("::RubyProf::WALL_TIME", 0) + end + # Use fresh config all for every example after { described_class.remove_instance_variable(:@config) } @@ -9,7 +15,7 @@ specify "defaults", :aggregate_failures do expect(subject.printer).to eq :flat - expect(subject.mode).to eq :wall + expect(subject.mode).to eq "wall" expect(subject.min_percent).to eq 1 expect(subject.include_threads).to eq false end @@ -39,10 +45,10 @@ end specify "with default config" do - expect(ruby_prof).to receive(:new).with( - merge_fibers: true, - include_threads: [Thread.current] - ).and_return(profile) + expect(ruby_prof).to receive(:new).with({ + include_threads: [Thread.current], + measure_mode: 0 + }).and_return(profile) expect(described_class.profile).to be_a(described_class::Report) end @@ -50,9 +56,7 @@ specify "with custom config" do described_class.config.include_threads = true - expect(ruby_prof).to receive(:new).with( - merge_fibers: true - ).and_return(profile) + expect(ruby_prof).to receive(:new).with({measure_mode: 0}).and_return(profile) described_class.profile end diff --git a/spec/test_prof/stack_prof_spec.rb b/spec/test_prof/stack_prof_spec.rb index e8618794..2f00a654 100644 --- a/spec/test_prof/stack_prof_spec.rb +++ b/spec/test_prof/stack_prof_spec.rb @@ -42,6 +42,18 @@ described_class.profile end + specify "with ignore_gc option" do + described_class.config.ignore_gc = true + + expect(stack_prof).to receive(:start).with( + mode: :wall, + raw: true, + ignore_gc: true + ) + + described_class.profile + end + specify "when block is given" do expect(stack_prof).to receive(:run).with( out: File.join(TestProf.config.output_dir, "stack-prof-report-wall-raw-stub.dump").to_s, @@ -61,6 +73,7 @@ end it "stops profiling and stores results" do + expect(described_class).to receive(:dump_json_report) expect(stack_prof).to receive(:results).with( File.join(TestProf.config.output_dir, "stack-prof-report-wall-raw-stub.dump").to_s ) diff --git a/test-prof.gemspec b/test-prof.gemspec index f4fb492d..9f50bdb3 100644 --- a/test-prof.gemspec +++ b/test-prof.gemspec @@ -24,18 +24,19 @@ Gem::Specification.new do |spec| "documentation_uri" => "https://test-prof.evilmartians.io/", "homepage_uri" => "https://test-prof.evilmartians.io/", "source_code_uri" => "https://github.com/test-prof/test-prof", - "funding_uri" => "https://github.com/sponsors/test-prof" + "funding_uri" => "https://github.com/sponsors/test-prof", + "default_lint_roller_plugin" => "RuboCop::TestProf::Plugin" } spec.files = Dir.glob("lib/**/*") + Dir.glob("config/**/*") + Dir.glob("assets/**/*") + %w[README.md LICENSE.txt CHANGELOG.md] spec.require_paths = ["lib"] - spec.required_ruby_version = ">= 2.5.0" + spec.required_ruby_version = ">= 2.7.0" spec.add_development_dependency "bundler", ">= 1.16" spec.add_development_dependency "rake", "~> 13.0" - spec.add_development_dependency "rspec", "~> 3.4" + spec.add_development_dependency "rspec-rails", ">= 4.0" spec.add_development_dependency "isolator", ">= 0.6" spec.add_development_dependency "minitest", ">= 5.9" spec.add_development_dependency "rubocop", ">= 0.77.0"