diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bc9e1d..1fd65f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -328,3 +328,12 @@ jobs: - uses: rubygems/release-gem@6317d8d1f7e28c24d28f6eff169ea854948bd9f7 # v1.2.0 with: token: ${{ steps.octo-sts.outputs.token }} + + complete: + name: Build (complete) + # Depends on the validation jobs, not on publish: publish is conditional + # (only on push to main) and would otherwise leave this job skipped on PRs. + needs: [validate-linux, validate-macos] + runs-on: ubuntu-24.04 + steps: + - run: echo "DONE!" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9466837..3eb41fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,6 +33,7 @@ jobs: complete: name: Main (complete) needs: + - build - test - nix runs-on: ubuntu-24.04 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index d3313ec..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish -on: workflow_dispatch - -concurrency: "publish" # Only one publish job at a time - -jobs: - publish-ruby: - name: Build and push gem to RubyGems.org - runs-on: ubuntu-24.04 - environment: "rubygems.org" # see: https://github.com/DataDog/libdatadog-rb/settings/environments - permissions: - id-token: write # Required for trusted publishing, see https://github.com/rubygems/release-gem - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set up Ruby - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # 1.310.0 - with: - ruby-version: "4.0" - bundler-cache: true - - name: Install dependencies - run: bundle install - - name: Fetch and extract prebuilt releases - run: bundle exec rake extract - - name: Package gems - run: bundle exec rake package - - uses: rubygems/release-gem@6317d8d1f7e28c24d28f6eff169ea854948bd9f7 # v1.2.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87740e6..14dc117 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,3 @@ jobs: bundler: ${{ matrix.ruby.bundler }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically - run: bundle exec rake - - run: bundle exec rake extract - if: ${{ matrix.ruby.version == '4.0' }} - - run: bundle exec rake package - if: ${{ matrix.ruby.version == '4.0' }} diff --git a/README.md b/README.md index 993de02..86817cd 100644 --- a/README.md +++ b/README.md @@ -14,23 +14,24 @@ You can also run `bundle exec pry` for an interactive prompt that will allow you ### Testing packaging locally -You can use `bundle exec rake package` to generate packages locally without publishing them. +First build the libdatadog binaries for your platform from source with +`bundle exec rake libdatadog:build`, then package them into gems (without +publishing) with `bundle exec rake gem:package`. -TIP: If the test that checks for permissions ("gem release process ... sets the right permissions on the gem files"), fails you -may need to run `umask 0022 && bundle exec rake package` so that the generated packages have the correct permissions. +You can check the file permissions of the built gems with +`bundle exec rake gem:validate`. ## Releasing a new version to rubygems.org Note: No Ruby needed to run this! It all runs in CI! 1. [ ] Locate the new libdatadog release on GitHub: -2. [ ] Update the `LIB_GITHUB_RELEASES` section of the `Rakefile` with the hashes from the new version -3. [ ] In the file: +2. [ ] In the file: - [ ] Update `LIB_VERSION` with the new version. Example: Setting "25.0.0" results in the first part of the string "25.0.0.1.0.x" - [ ] (OPTIONAL) Update `GEM_PRERELEASE_VERSION` with a prerelease descriptor. This is only needed if you want to do a prerelease. Example: Setting ".beta" results in "25.0.0.1.0.beta". -4. [ ] Commit change, open PR, get it merged -5. [ ] Trigger the "Publish" workflow in -6. [ ] Verify that release shows up correctly on: +3. [ ] Commit change, open PR, get it merged +4. [ ] Trigger the "Build" workflow on `main` with the "Push gems to RubyGems.org" option enabled: +5. [ ] Verify that release shows up correctly on: ## Contributing diff --git a/Rakefile b/Rakefile index 54e5e7b..31ea972 100644 --- a/Rakefile +++ b/Rakefile @@ -1,237 +1,6 @@ # frozen_string_literal: true require "bundler/gem_tasks" -require "rspec/core/rake_task" -require "standard/rake" unless RUBY_VERSION < "2.6" -require "fileutils" -require "http" unless RUBY_VERSION < "2.5" -require "pry" -require "rubygems/package" - -RSpec::Core::RakeTask.new(:spec) - -# Note: When packaging rc releases and the like, you may need to set this differently from LIB_VERSION -LIB_VERSION_TO_PACKAGE = Libdatadog::LIB_VERSION - -unless LIB_VERSION_TO_PACKAGE.start_with?(Libdatadog::LIB_VERSION) - raise "`LIB_VERSION_TO_PACKAGE` setting in (#{LIB_VERSION_TO_PACKAGE}) does not match " \ - "`LIB_VERSION` setting in (#{Libdatadog::LIB_VERSION})" -end - -LIB_GITHUB_RELEASES = [ - { - file: "libdatadog-aarch64-alpine-linux-musl.tar.gz", - sha256: "e93457fd251444dbece87ad424140711a43cbc20da2c5c98009ba84d9be5e733", - ruby_platform: "aarch64-linux-musl" - }, - { - file: "libdatadog-aarch64-unknown-linux-gnu.tar.gz", - sha256: "2a11baa8966e6c681b591124dfe6f4770fc4eae2c56f78aa2c7948088aea7a7b", - ruby_platform: "aarch64-linux" - }, - { - file: "libdatadog-x86_64-alpine-linux-musl.tar.gz", - sha256: "42ec78865aacb9259656834dec9c4ab16ffe3d654c7eb5286eb543c7fd8f8baf", - ruby_platform: "x86_64-linux-musl" - }, - { - file: "libdatadog-x86_64-unknown-linux-gnu.tar.gz", - sha256: "2925f2ed001ecb0d34d7c485aa5aadf86678bba62012ccb3fd5bdf272faee5da", - ruby_platform: "x86_64-linux" - }, - { - file: "libdatadog-aarch64-apple-darwin.tar.gz", - sha256: "d277945d612fa6f38f644f15d1c1b2f63ea92cd53794e9d48ee7ad3290a7660d", - ruby_platform: "arm64-darwin" - } -] - -task default: [ - :spec, - (:standard unless RUBY_VERSION < "2.6") -].compact - -desc "Download lib release from github" -task :fetch do - Helpers.each_github_release_variant do |file:, sha256:, target_directory:, target_file:, **_| - target_url = "https://github.com/datadog/libdatadog/releases/download/v#{LIB_VERSION_TO_PACKAGE}/#{file}" - - if File.exist?(target_file) - target_file_hash = Digest::SHA256.hexdigest(File.read(target_file)) - - if target_file_hash == sha256 - puts "Found #{target_file} matching the expected sha256, skipping download" - next - else - puts "Found #{target_file} with hash (#{target_file_hash}) BUT IT DID NOT MATCH THE EXPECTED sha256 (#{sha256}), downloading it again..." - end - end - - puts "Going to download #{target_url} into #{target_file}" - - File.open(target_file, "wb") do |file| - HTTP.follow.get(target_url).body.each { |chunk| file.write(chunk) } - end - - if Digest::SHA256.hexdigest(File.read(target_file)) == sha256 - puts "Success!" - else - raise "Downloaded file is corrupt, does not match expected sha256" - end - end -end - -desc "Extract lib downloaded releases" -task extract: [:fetch] do - Helpers.each_github_release_variant do |target_directory:, target_file:, **_| - puts "Extracting #{target_file}" - File.open(target_file, "rb") do |file| - Gem::Package.new("").extract_tar_gz(file, target_directory) - end - - # Fix file permissions after extraction - puts "Fixing file permissions in #{target_directory}" - Helpers.fix_file_permissions(target_directory) - end -end - -desc "Package lib downloaded releases as gems" -task package: [ - :spec, - (:"standard:fix" unless RUBY_VERSION < "2.6") -] do - gemspec = eval(File.read("libdatadog.gemspec"), nil, "libdatadog.gemspec") # standard:disable Security/Eval - FileUtils.mkdir_p("pkg") - - # Fallback package with all binaries - # This package will get used by (1) platforms that have no matching `ruby_platform` or (2) that have set - # "BUNDLE_FORCE_RUBY_PLATFORM" (or its equivalent via code) to avoid precompiled gems. - # In a previous version of libdatadog, this package had no binaries, but that could mean that we broke customers in case (2). - # For customers in case (1), this package is a no-op, and dd-trace-rb will correctly detect and warn that - # there are no valid binaries for the platform. - Helpers.package_for(gemspec, ruby_platform: nil, files: Helpers.files_for("x86_64-linux", "x86_64-linux-musl", "aarch64-linux", "aarch64-linux-musl", "arm64-darwin")) - - # We include both glibc and musl variants in the same binary gem to avoid the issues - # documented in https://github.com/rubygems/rubygems/issues/3174 - Helpers.package_for(gemspec, ruby_platform: "x86_64-linux", files: Helpers.files_for("x86_64-linux", "x86_64-linux-musl")) - Helpers.package_for(gemspec, ruby_platform: "aarch64-linux", files: Helpers.files_for("aarch64-linux", "aarch64-linux-musl")) - - # macOS package (Apple Silicon) - Helpers.package_for(gemspec, ruby_platform: "arm64-darwin", files: Helpers.files_for("arm64-darwin")) -end - -Rake::Task["package"].enhance { Rake::Task["spec_validate_permissions"].execute } - -task :spec_validate_permissions do - require "rspec" - RSpec.world.reset # If any other tests ran before, flushes them - ret = RSpec::Core::Runner.run(["spec/gem_packaging.rb"]) - raise "Release tests failed! See error output above." if ret != 0 -end - -desc "Release all packaged gems" -task push_to_rubygems: [ - :"release:guard_clean" -] do - [ - "gem push pkg/libdatadog-#{Libdatadog::VERSION}.gem", - "gem push pkg/libdatadog-#{Libdatadog::VERSION}-x86_64-linux.gem", - "gem push pkg/libdatadog-#{Libdatadog::VERSION}-aarch64-linux.gem", - "gem push pkg/libdatadog-#{Libdatadog::VERSION}-arm64-darwin.gem" - ].each do |command| - puts "Running: #{command}" - abort unless system(command) - end -end - -module Helpers - # Files that should have executable permissions (755) in the gem - # Note: .so for Linux, .dylib for macOS - EXECUTABLE_FILES = ["libdatadog-crashtracking-receiver", "libdatadog_profiling.so", "libdatadog_profiling.dylib"].freeze - - def self.each_github_release_variant - LIB_GITHUB_RELEASES.each do |variant| - file = variant.fetch(:file) - sha256 = variant.fetch(:sha256) - ruby_platform = variant.fetch(:ruby_platform) - - # These two are so common that we just centralize them here - target_directory = "vendor/libdatadog-#{Libdatadog::LIB_VERSION}/#{ruby_platform}" - target_file = "#{target_directory}/#{file}" - - FileUtils.mkdir_p(target_directory) - - yield(file: file, sha256: sha256, ruby_platform: ruby_platform, target_directory: target_directory, target_file: target_file) - end - end - - def self.package_for(gemspec, ruby_platform:, files:) - target_gemspec = gemspec.dup - target_gemspec.files += files - target_gemspec.platform = ruby_platform if ruby_platform - - puts "Building with ruby_platform=#{ruby_platform.inspect} including: (this can take a while)" - pp target_gemspec.files - - package = Gem::Package.build(target_gemspec) - FileUtils.mv(package, "pkg") - puts("-" * 80) - end - - def self.fix_file_permissions(directory) - Dir.glob("#{directory}/**/*").each do |path| - next unless File.file?(path) - - filename = File.basename(path) - current_permissions = File.stat(path).mode & 0o777 - - if EXECUTABLE_FILES.include?(filename) - # Should be executable (755), fix if not - if current_permissions != 0o755 - puts "Fixing permissions for #{filename}: #{current_permissions.to_s(8)} -> 755" - FileUtils.chmod(0o755, path) - end - elsif current_permissions != 0o644 - # Should be non-executable (644), fix if not - puts "Fixing permissions for #{filename}: #{current_permissions.to_s(8)} -> 644" - FileUtils.chmod(0o644, path) - end - end - end - - def self.files_for( - *included_platforms, - excluded_files: [ - "datadog_profiling.pc", # we use the datadog_profiling_with_rpath.pc variant - "libdatadog_profiling.a", "datadog_profiling-static.pc", # We don't use the static library - "libdatadog_profiling.debug", # We don't include debug info - "DatadogConfig.cmake" # We don't compile using cmake - ] - ) - files = [] - - each_github_release_variant do |ruby_platform:, target_directory:, target_file:, **_| - next unless included_platforms.include?(ruby_platform) - - downloaded_release_tarball = target_file - - files += - Dir.glob("#{target_directory}/**/*") - .select { |path| File.file?(path) } - .reject { |path| path == downloaded_release_tarball } - .reject { |path| excluded_files.include?(File.basename(path)) } - end - - files - end -end - -Rake::Task["build"].clear -task(:build) { raise "Build task is disabled, use package instead" } - -Rake::Task["release"].clear -task(:release) { Rake::Task["push_to_rubygems"].invoke } - -# Load additional tasks +# Each logical group of tasks lives in its own file under tasks/. Dir.glob("tasks/**/*.rake").each { |r| import r } diff --git a/spec/gem_packaging.rb b/spec/gem_packaging.rb deleted file mode 100644 index 86c4601..0000000 --- a/spec/gem_packaging.rb +++ /dev/null @@ -1,51 +0,0 @@ -# Note: This file does not end with _spec on purpose, it should only be run after packaging, e.g. with `rake spec_validate_permissions` - -require "rubygems" -require "rubygems/package" -require "rubygems/package/tar_reader" -require "libdatadog" -require "zlib" - -RSpec.describe "gem release process (after packaging)" do - let(:gem_version) { Libdatadog::VERSION } - let(:packaged_gem_file) { "pkg/libdatadog-#{gem_version}.gem" } - - it "sets the right permissions on the .gem files" do - gem_files = Dir.glob("pkg/*.gem") - expect(gem_files).to include(packaged_gem_file) - - gem_files.each do |gem_file| - Gem::Package::TarReader.new(File.open(gem_file)) do |tar| - data = tar.find { |entry| entry.header.name == "data.tar.gz" } - - Gem::Package::TarReader.new(Zlib::GzipReader.new(StringIO.new(data.read))) do |data_tar| - data_tar.each do |entry| - filename = entry.header.name.split("/").last - octal_permissions = entry.header.mode.to_s(8)[-3..-1] - - expected_permissions = Helpers::EXECUTABLE_FILES.include?(filename) ? "755" : "644" - - expect(octal_permissions).to eq(expected_permissions), - "Unexpected permissions for #{filename} inside #{gem_file} (got #{octal_permissions}, " \ - "expected #{expected_permissions})" - end - end - end - end - end - - it "prefixes all public symbols in .so files" do - so_files = Dir.glob("vendor/libdatadog-#{Libdatadog::LIB_VERSION}/**/*.so") - expect(so_files.size).to be 4 - - so_files.each do |so_file| - raw_symbols = `nm -D --defined-only #{so_file}` - - symbols = raw_symbols.split("\n").map { |symbol| symbol.split(" ").last.downcase }.sort - expect(symbols.size).to be > 20 # Quick sanity check - expect(symbols).to all( - start_with("ddog_").or(start_with("blaze_")) - ) - end - end -end diff --git a/tasks/build.rake b/tasks/build.rake index dfe1300..32aa779 100644 --- a/tasks/build.rake +++ b/tasks/build.rake @@ -1,6 +1,7 @@ # frozen_string_literal: true require "etc" +require "fileutils" require "pathname" module BuildFromSource @@ -80,6 +81,31 @@ module BuildFromSource end end + module Permissions + # Files that must be executable (0755) in the gem; everything else 0644. + # Note: .so for Linux, .dylib for macOS. + EXECUTABLE_FILES = %w[ + libdatadog-crashtracking-receiver + libdatadog_profiling.so + libdatadog_profiling.dylib + ].freeze + + # Normalise file permissions of the built artifacts before packaging. + def self.fix(directory) + Dir.glob("#{directory}/**/*").each do |path| + next unless File.file?(path) + + filename = File.basename(path) + current = File.stat(path).mode & 0o777 + expected = EXECUTABLE_FILES.include?(filename) ? 0o755 : 0o644 + next if current == expected + + puts "Fixing permissions for #{filename}: #{current.to_s(8)} -> #{expected.to_s(8)}" + FileUtils.chmod(expected, path) + end + end + end + module Builder class << self # Build the cargo install command for the builder crate's `release` binary. @@ -160,7 +186,7 @@ namespace :libdatadog do system(env, paths.builder_bin.to_s, "--out", target_dir.to_s) || raise("Builder failed") # Fix file permissions to match expected values for packaging - Helpers.fix_file_permissions(target_dir.to_s) + BuildFromSource::Permissions.fix(target_dir) puts "Done! Artifacts in #{target_dir}" end diff --git a/tasks/gem.rake b/tasks/gem.rake index a3d8521..062c531 100644 --- a/tasks/gem.rake +++ b/tasks/gem.rake @@ -7,10 +7,6 @@ require "zlib" require "stringio" # Builds .gem packages from vendored libdatadog artifacts. -# -# Intentionally duplicates some Rakefile logic so both can coexist during -# the migration; the Rakefile originals will be removed once the new tasks -# are fully adopted. module GemPackaging module Platform # Mapping from gem platform names to the vendor platform directories diff --git a/tasks/release.rake b/tasks/release.rake new file mode 100644 index 0000000..a2a8fae --- /dev/null +++ b/tasks/release.rake @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# `rake build` (from bundler/gem_tasks) would build a binary-less gem straight +# from the gemspec. Real gems bundle the platform-specific libdatadog artifacts +# and are built via `rake gem:package`, so disable the default to avoid mistakes. +Rake::Task["build"].clear +task(:build) { raise "Build task is disabled, use gem:package instead" } + +# `rake release` (from bundler/gem_tasks) tags and pushes; here releasing just +# pushes the already-built platform gems from pkg/ (the version tag lives in the +# libdatadog-rb repo, the binaries are built and packaged by CI). +Rake::Task["release"].clear +task(:release) { Rake::Task["push_to_rubygems"].invoke } + +desc "Push all built gems in pkg/ to RubyGems.org" +task push_to_rubygems: [ + :"release:guard_clean" +] do + [ + "gem push pkg/libdatadog-#{Libdatadog::VERSION}.gem", + "gem push pkg/libdatadog-#{Libdatadog::VERSION}-x86_64-linux.gem", + "gem push pkg/libdatadog-#{Libdatadog::VERSION}-aarch64-linux.gem", + "gem push pkg/libdatadog-#{Libdatadog::VERSION}-arm64-darwin.gem" + ].each do |command| + puts "Running: #{command}" + abort unless system(command) + end +end diff --git a/tasks/spec.rake b/tasks/spec.rake new file mode 100644 index 0000000..c3fb378 --- /dev/null +++ b/tasks/spec.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "rspec/core/rake_task" +require "standard/rake" unless RUBY_VERSION < "2.6" + +RSpec::Core::RakeTask.new(:spec) + +task default: [ + :spec, + (:standard unless RUBY_VERSION < "2.6") +].compact