diff --git a/.ruby-version b/.ruby-version index 6cb9d3dd0d..f9892605c7 100755 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.4.3 +3.4.4 diff --git a/Gemfile b/Gemfile index 40ca2189b9..47c474e4af 100755 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,5 @@ source "https://rubygems.org" -ruby "~> 3.4.3" +ruby "~> 3.4.4" gem "rails", "~> 6" gem "active_model_serializers" diff --git a/Gemfile.lock b/Gemfile.lock index 17755a1c50..1261d23699 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -145,7 +145,7 @@ GEM google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.3.0) + google-cloud-env (2.3.1) base64 (~> 0.2) faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) @@ -173,7 +173,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.12.0) + json (2.12.2) jsonapi-renderer (0.2.2) jwt (2.10.1) base64 @@ -380,7 +380,7 @@ GEM crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.9.1) - websocket-driver (0.7.7) + websocket-driver (0.8.0) base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -438,7 +438,7 @@ DEPENDENCIES webmock RUBY VERSION - ruby 3.4.3p32 + ruby 3.4.4p34 BUNDLED WITH 2.6.9 diff --git a/app/controllers/api/images_controller.rb b/app/controllers/api/images_controller.rb index 28df5f6a6f..289f05def7 100644 --- a/app/controllers/api/images_controller.rb +++ b/app/controllers/api/images_controller.rb @@ -5,7 +5,6 @@ module Api # 3. Image is transferred to the "trusted bucket". class ImagesController < Api::AbstractController cattr_accessor :store_locally - self.store_locally = !ENV["GCS_BUCKET"] def create mutate Images::Create.run(raw_json, device: current_device) @@ -32,8 +31,12 @@ def storage_auth private + def self.store_locally? + !ENV["GCS_BUCKET"] + end + def policy_class - if ImagesController.store_locally + if ImagesController.store_locally? Images::StubPolicy else Images::GeneratePolicy diff --git a/app/models/image.rb b/app/models/image.rb index 22405d9a40..080b5f55c4 100644 --- a/app/models/image.rb +++ b/app/models/image.rb @@ -26,22 +26,23 @@ def set_defaults CONFIG = { default_url: DEFAULT_URL, styles: RMAGICK_STYLES, size: { in: 0..MAX_IMAGE_SIZE } } - BUCKET = ENV["GCS_BUCKET"] - - ROOT_PATH = BUCKET ? - "https://#{BUCKET}.storage.googleapis.com" : "/system" - IMAGE_URL_TPL = - ROOT_PATH + "/images/attachments/%{chunks}/%{size}/%{filename}?%{timestamp}" CONTENT_TYPES = ["image/jpg", "image/jpeg", "image/png", "image/gif"] GCS_ACCESS_KEY_ID = ENV.fetch("GCS_KEY") { puts "Not using Google Cloud" } - GCS_HOST = "http://#{BUCKET}.storage.googleapis.com" GCS_SECRET_ACCESS_KEY = ENV.fetch("GCS_ID") { puts "Not using Google Cloud" } # Worst case scenario for 1280x1280 BMP. - GCS_BUCKET_NAME = ENV["GCS_BUCKET"] has_one_attached :attachment + def bucket + ENV["GCS_BUCKET"] + end + + def image_url_tpl + root_path = bucket ? "https://#{bucket}.storage.googleapis.com" : "/system" + root_path + "/images/attachments/%{chunks}/%{size}/%{filename}?%{timestamp}" + end + def set_attachment_by_url(url) io = URI.parse(url).open fname = "image_#{self.id}" @@ -73,9 +74,8 @@ def regular_image? end def regular_url - if BUCKET - # Not sure why. TODO: Investigate why Rails URL helpers don't work here. - "https://storage.googleapis.com/#{BUCKET}/#{attachment.key}" + if bucket + "https://storage.googleapis.com/#{bucket}/#{attachment.key}" else Rails .application @@ -86,7 +86,7 @@ def regular_url end def legacy_url(size) - url = IMAGE_URL_TPL % { + url = image_url_tpl % { chunks: id.to_s.rjust(9, "0").scan(/.{3}/).join("/"), filename: attachment_file_name, size: size, @@ -108,7 +108,7 @@ def attachment_url(size = "x640") end def self.self_hosted_image_upload(key:, file:) - raise "No." unless Api::ImagesController.store_locally + raise "No." unless Api::ImagesController.store_locally? name = key.split("/").last src = file.tempfile.path dest = File.join("public", "direct_upload", "temp", name) diff --git a/app/mutations/images/generate_policy.rb b/app/mutations/images/generate_policy.rb index 1f2892a867..d5dd201b62 100644 --- a/app/mutations/images/generate_policy.rb +++ b/app/mutations/images/generate_policy.rb @@ -3,9 +3,6 @@ module Images class GeneratePolicy < Mutations::Command - BUCKET_NAME = ENV.fetch("GCS_BUCKET") { "YOU_MUST_CONFIG_GOOGLE_CLOUD_STORAGE" } - JSON_KEY = ENV["GOOGLE_CLOUD_KEYFILE_JSON"] - BUCKET = JSON_KEY && Google::Cloud::Storage.new.bucket(BUCKET_NAME) HMM = "GCS NOT SETUP!" # # Is there a better way to reach in and grab the ActiveStorage configs? # CONFIG = YAML.load(ERB.new(File.read("config/storage.yml")).result(binding)).fetch("google") @@ -13,7 +10,7 @@ class GeneratePolicy < Mutations::Command def execute { verb: "POST", - url: "//storage.googleapis.com/#{BUCKET_NAME || HMM}/", + url: "//storage.googleapis.com/#{bucket_name || HMM}/", form_data: { "key" => file_path, "acl" => "public-read", @@ -31,16 +28,25 @@ def execute private + def bucket_name + ENV["GCS_BUCKET"] + end + + def bucket + json_key = ENV["GOOGLE_CLOUD_KEYFILE_JSON"] + json_key && Google::Cloud::Storage.new.bucket(bucket_name) + end + def post_object - @post_object ||= BUCKET ? - BUCKET.post_object(file_path, policy: policy).fields : {} + @post_object ||= bucket ? + bucket.post_object(file_path, policy: policy).fields : {} end def policy @policy ||= { expiration: (Time.now + 1.hour).utc.xmlschema, conditions: [ - { bucket: BUCKET_NAME }, + { bucket: bucket_name }, { key: file_path }, { acl: "public-read" }, [:eq, "$Content-Type", "image/jpeg"], diff --git a/config/application.rb b/config/application.rb index 4372fcfbba..1ee18cd100 100755 --- a/config/application.rb +++ b/config/application.rb @@ -2,6 +2,7 @@ require File.expand_path("../boot", __FILE__) require_relative "../app/lib/celery_script/cs_heap" require "rails/all" +require_relative "./config_helpers/active_storage" # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. @@ -12,8 +13,6 @@ class Application < Rails::Application Delayed::Worker.max_attempts = 4 REDIS_ENV_KEY = ENV.fetch("WHERE_IS_REDIS_URL", "REDIS_URL") REDIS_URL = ENV.fetch(REDIS_ENV_KEY, "redis://redis:6379/0") - gcs_enabled = - %w[ GOOGLE_CLOUD_KEYFILE_JSON GCS_PROJECT GCS_BUCKET ].all? { |s| ENV.key? s } config.lograge.enabled = true config.lograge.ignore_actions = [ "Api::RmqUtilsController#user_action", @@ -22,8 +21,7 @@ class Application < Rails::Application "Api::RmqUtilsController#topic_action", ] config.load_defaults 6.0 - config.active_storage.service = gcs_enabled ? - :google : :local + config.active_storage.service = ConfigHelpers::ActiveStorage.service config.cache_store = :redis_cache_store, { url: REDIS_URL, ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE } } config.middleware.use Rack::Attack config.active_record.schema_format = :sql diff --git a/config/config_helpers/active_storage.rb b/config/config_helpers/active_storage.rb new file mode 100644 index 0000000000..b6e13121d1 --- /dev/null +++ b/config/config_helpers/active_storage.rb @@ -0,0 +1,13 @@ +module ConfigHelpers + module ActiveStorage + REQUIRED_KEYS = %w[ + GOOGLE_CLOUD_KEYFILE_JSON + GCS_PROJECT + GCS_BUCKET + ].freeze + + def self.service + REQUIRED_KEYS.all? { |key| ENV.key?(key) } ? :google : :local + end + end +end diff --git a/docker_configs/api.Dockerfile b/docker_configs/api.Dockerfile index 74d907a913..f5496a05c6 100644 --- a/docker_configs/api.Dockerfile +++ b/docker_configs/api.Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:3.4.3 +FROM ruby:3.4.4 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null && \ sh -c '. /etc/os-release; echo $VERSION_CODENAME; echo "deb http://apt.postgresql.org/pub/repos/apt/ $VERSION_CODENAME-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' && \ apt-get update -qq && apt-get install -y build-essential libpq-dev postgresql postgresql-contrib && \ diff --git a/frontend/three_d_garden/bot/components/__tests__/tools_test.tsx b/frontend/three_d_garden/bot/components/__tests__/tools_test.tsx index 448a64a2fb..ce00b22c62 100644 --- a/frontend/three_d_garden/bot/components/__tests__/tools_test.tsx +++ b/frontend/three_d_garden/bot/components/__tests__/tools_test.tsx @@ -90,8 +90,10 @@ describe("", () => { toolSlot4.body.pullout_direction = ToolPulloutDirection.NEGATIVE_Y; const toolSlot5 = fakeToolSlot(); toolSlot5.body.tool_id = tool5.body.id; + toolSlot5.body.gantry_mounted = true; const toolSlot6 = fakeToolSlot(); toolSlot6.body.tool_id = tool6.body.id; + toolSlot6.body.gantry_mounted = true; p.toolSlots = [ { toolSlot: toolSlot0, tool: tool0 }, { toolSlot: toolSlot1, tool: tool1 }, diff --git a/frontend/three_d_garden/bot/components/tools.tsx b/frontend/three_d_garden/bot/components/tools.tsx index 8407c36452..37fd4fe7b8 100644 --- a/frontend/three_d_garden/bot/components/tools.tsx +++ b/frontend/three_d_garden/bot/components/tools.tsx @@ -82,6 +82,7 @@ interface ConvertedTools { toolName: string | undefined; toolPulloutDirection: ToolPulloutDirection; firstTrough?: boolean; + gantryMounted?: boolean; } export const convertSlotsWithTools = @@ -98,6 +99,7 @@ export const convertSlotsWithTools = toolName, toolPulloutDirection: swt.toolSlot.body.pullout_direction, firstTrough: troughIndex < 2, + gantryMounted: swt.toolSlot.body.gantry_mounted, }; }); }; @@ -423,6 +425,7 @@ export const Tools = (props: ToolsProps) => { {tools.map((tool, i) => )} ; }; diff --git a/package.json b/package.json index 72961ac82f..81a515524e 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,8 @@ "@blueprintjs/core": "5.19.0", "@blueprintjs/select": "5.3.20", "@monaco-editor/react": "4.7.0", - "@parcel/transformer-sass": "2.15.1", - "@parcel/transformer-typescript-tsc": "2.15.1", + "@parcel/transformer-sass": "2.15.2", + "@parcel/transformer-typescript-tsc": "2.15.2", "@react-spring/three": "10.0.1", "@react-three/drei": "9.122.0", "@react-three/fiber": "8.18.0", @@ -62,7 +62,7 @@ "delaunator": "5.0.1", "events": "3.3.0", "farmbot": "15.8.11", - "i18next": "25.2.0", + "i18next": "25.2.1", "lodash": "4.17.21", "markdown-it": "14.1.0", "markdown-it-emoji": "3.0.0", @@ -70,7 +70,7 @@ "monaco-editor": "0.52.2", "mqtt": "5.13.0", "npm": "11.4.1", - "parcel": "2.15.1", + "parcel": "2.15.2", "process": "0.11.10", "promise-timeout": "1.3.0", "punycode": "1.4.1", @@ -79,7 +79,7 @@ "react-color": "2.19.3", "react-dom": "18.3.1", "react-redux": "9.2.0", - "react-router": "7.6.0", + "react-router": "7.6.1", "redux": "5.0.1", "redux-immutable-state-invariant": "2.1.0", "redux-thunk": "3.1.0", diff --git a/spec/config_helpers/active_storage_spec.rb b/spec/config_helpers/active_storage_spec.rb new file mode 100644 index 0000000000..6110d25ec2 --- /dev/null +++ b/spec/config_helpers/active_storage_spec.rb @@ -0,0 +1,23 @@ +require "spec_helper" + +describe ConfigHelpers::ActiveStorage do + it "returns :google when all required env vars are set" do + with_modified_env( + GOOGLE_CLOUD_KEYFILE_JSON: "key", + GCS_PROJECT: "project", + GCS_BUCKET: "bucket", + ) do + expect(described_class.service).to eq(:google) + end + end + + it "returns :local when no required env vars are set" do + with_modified_env( + GOOGLE_CLOUD_KEYFILE_JSON: nil, + GCS_PROJECT: nil, + GCS_BUCKET: nil, + ) do + expect(described_class.service).to eq(:local) + end + end +end diff --git a/spec/controllers/api/farmware_installations/farmware_installations_controller_spec.rb b/spec/controllers/api/farmware_installations/farmware_installations_controller_spec.rb index 50af6dbea0..a7edc53517 100644 --- a/spec/controllers/api/farmware_installations/farmware_installations_controller_spec.rb +++ b/spec/controllers/api/farmware_installations/farmware_installations_controller_spec.rb @@ -7,7 +7,9 @@ describe "#create" do it "creates a new FarmwareInstallation" do sign_in user - url = Faker::Internet.url host: "example.com", path: "/#{SecureRandom.hex(16)}/manifest.json" + hex = SecureRandom.hex(16) + url = Faker::Internet.url host: "example.com", path: "/#{hex}/manifest.json" + stub_request(:get, url).to_return(status: 200, body: "", headers: {}) payload = { url: url } old_installation_count = FarmwareInstallation.count post :create, body: payload.to_json, params: { format: :json } diff --git a/spec/controllers/api/images/images_spec.rb b/spec/controllers/api/images/images_spec.rb index 70029a2eaa..8979ad7d06 100644 --- a/spec/controllers/api/images/images_spec.rb +++ b/spec/controllers/api/images/images_spec.rb @@ -1,40 +1,52 @@ require "spec_helper" -WebMock.allow_net_connect! describe Api::ImagesController do include Devise::Test::ControllerHelpers let(:user) { FactoryBot.create(:user) } - it "" do - fake_file = ActionDispatch::Http::UploadedFile.new(filename: "wow.jpg", type: "image/jpg", head: "", tempfile: Tempfile.new) + it "uploads file" do + fake_file = ActionDispatch::Http::UploadedFile.new( + filename: "wow.jpg", + type: "image/jpg", + head: "", + tempfile: Tempfile.new, + ) name = "wow.jpg" Image.self_hosted_image_upload(key: "/abc.jpg", file: fake_file) expected = "public/direct_upload/temp/abc.jpg" - assert File.file?(expected) - File.delete(expected) + begin + assert File.file?(expected) + ensure + File.delete(expected) if File.exist?(expected) + end end it "Creates a policy object" do - sign_in user - b4 = Api::ImagesController.store_locally - Api::ImagesController.store_locally = false - get :storage_auth - Api::ImagesController.store_locally = b4 + allow(Google::Cloud::Storage).to receive_message_chain("new.bucket.post_object.fields") + .and_return({ signature: "signature" }) - expect(response.status).to eq(200) - expect(json).to be_kind_of(Hash) - expect(json[:verb]).to eq("POST") - expect(json[:url]).to include("googleapis") - expect(json[:form_data].keys.sort).to include(:signature) - expect(json[:instructions]).to include("POST the resulting URL as an 'attachment_url'") + with_modified_env( + GOOGLE_CLOUD_KEYFILE_JSON: "key", + GCS_BUCKET: "bucket", + ) do + + sign_in user + get :storage_auth + + expect(response.status).to eq(200) + expect(json).to be_kind_of(Hash) + expect(json[:verb]).to eq("POST") + expect(json[:url]).to include("googleapis") + expect(json[:form_data].keys.sort).to include(:signature) + expect(json[:form_data][:signature]).to eq("signature") + expect(json[:instructions]).to include("POST the resulting URL as an 'attachment_url'") + end end it "Creates a (stub) policy object" do sign_in user - b4 = Api::ImagesController.store_locally - Api::ImagesController.store_locally = true get :storage_auth - Api::ImagesController.store_locally = b4 + expect(response.status).to eq(200) expect(json).to be_kind_of(Hash) expect(json[:verb]).to eq("POST") @@ -72,7 +84,17 @@ end describe "#create" do + image_data = File.read(Rails.root.join("public", "plant.jpg")) + it "creates one image", :slow do + stub_request(:get, FAKE_ATTACHMENT_URL).to_return( + status: 200, + body: image_data, + headers: { + "Content-Type" => "image/jpeg", + "Content-Length" => image_data.size.to_s + } + ) sign_in user before_count = Image.count post :create, @@ -88,18 +110,18 @@ expect(json.dig :meta, :y).to eq(nil) expect(json.dig :meta, :z).to eq(3) end + end - describe "#delete" do - it "deletes an image" do - sign_in user - image = FactoryBot.create(:image, device: user.device) - before_count = Image.count - run_jobs_now do - delete :destroy, params: { id: image.id } - end - expect(response.status).to eq(200) - expect(Image.count).to be < before_count + describe "#delete" do + it "deletes an image" do + sign_in user + image = FactoryBot.create(:image, device: user.device) + before_count = Image.count + run_jobs_now do + delete :destroy, params: { id: image.id } end + expect(response.status).to eq(200) + expect(Image.count).to be < before_count end end end diff --git a/spec/models/image_spec.rb b/spec/models/image_spec.rb index 3a161d3670..c000da1bc4 100644 --- a/spec/models/image_spec.rb +++ b/spec/models/image_spec.rb @@ -1,10 +1,18 @@ require "spec_helper" -WebMock.allow_net_connect! describe Image do let(:device) { FactoryBot.create(:device) } + image_data = File.read(Rails.root.join("public", "plant.jpg")) it "adds URL attachments", :slow do + stub_request(:get, FAKE_ATTACHMENT_URL).to_return( + status: 200, + body: image_data, + headers: { + "Content-Type" => "image/jpeg", + "Content-Length" => image_data.size.to_s + } + ) image = Image.create(device: device) expect(image.attachment_processed_at).to be_nil expect(image.attachment.attached?).to be false @@ -27,7 +35,7 @@ end it "generates a URL when BUCKET is set" do - const_reassign(Image, :BUCKET, "foo") do + with_modified_env(GCS_BUCKET: "foo") do i = Image.new expect(i).to receive(:attachment).and_return(Struct.new(:key).new("bar")) url = i.regular_url diff --git a/spec/mutations/images/generate_policy_spec.rb b/spec/mutations/images/generate_policy_spec.rb index 70df92e7fe..95d48d92ca 100644 --- a/spec/mutations/images/generate_policy_spec.rb +++ b/spec/mutations/images/generate_policy_spec.rb @@ -2,21 +2,50 @@ describe Images::GeneratePolicy do it "has a policy object (Hash)" do - policy = Images::GeneratePolicy.new.send(:policy) - expiration = Time.parse(policy.fetch(:expiration)) - one_hour = (Time.now + 1.hour).utc - time_diff = (one_hour - expiration).round - expect(time_diff).to be >= 0 - expect(time_diff).to be <= 1 + with_modified_env(GCS_BUCKET: "") do + policy = Images::GeneratePolicy.new.send(:policy) + expiration = Time.parse(policy.fetch(:expiration)) + one_hour = (Time.now + 1.hour).utc + time_diff = (one_hour - expiration).round + expect(time_diff).to be >= 0 + expect(time_diff).to be <= 1 - conditions = policy.fetch(:conditions).map(&:to_a).map(&:flatten) - { - 0 => eq([:bucket, "YOU_MUST_CONFIG_GOOGLE_CLOUD_STORAGE"]), - 2 => eq([:acl, "public-read"]), - 3 => eq([:eq, "$Content-Type", "image/jpeg"]), - 4 => eq(["content-length-range", 1, 7340032]), - }.map do |(index, meet_expectation)| - expect(conditions[index]).to meet_expectation + conditions = policy.fetch(:conditions).map(&:to_a).map(&:flatten) + { + 0 => eq([:bucket, ""]), + 2 => eq([:acl, "public-read"]), + 3 => eq([:eq, "$Content-Type", "image/jpeg"]), + 4 => eq(["content-length-range", 1, 7340032]), + }.map do |(index, meet_expectation)| + expect(conditions[index]).to meet_expectation + end + end + end + + it "has a policy object (GCS)" do + allow(Google::Cloud::Storage).to receive(:new) + .and_return(double(bucket: double())) + + with_modified_env( + GOOGLE_CLOUD_KEYFILE_JSON: "key", + GCS_BUCKET: "gcs", + ) do + policy = Images::GeneratePolicy.new.send(:policy) + expiration = Time.parse(policy.fetch(:expiration)) + one_hour = (Time.now + 1.hour).utc + time_diff = (one_hour - expiration).round + expect(time_diff).to be >= 0 + expect(time_diff).to be <= 1 + + conditions = policy.fetch(:conditions).map(&:to_a).map(&:flatten) + { + 0 => eq([:bucket, "gcs"]), + 2 => eq([:acl, "public-read"]), + 3 => eq([:eq, "$Content-Type", "image/jpeg"]), + 4 => eq(["content-length-range", 1, 7340032]), + }.map do |(index, meet_expectation)| + expect(conditions[index]).to meet_expectation + end end end end diff --git a/spec/mutations/sequences/show_spec.rb b/spec/mutations/sequences/show_spec.rb new file mode 100644 index 0000000000..1453c7f40d --- /dev/null +++ b/spec/mutations/sequences/show_spec.rb @@ -0,0 +1,22 @@ +require "spec_helper" + +describe Sequences::Show do + let(:user) { FactoryBot.create(:user) } + let(:sequence) { FakeSequence.create(device: user.device) } + + describe "#sequence_version" do + it "returns the associated sequence_version if association is loaded" do + allow(sequence.association(:sequence_version)).to receive(:loaded?).and_return(true) + instance = Sequences::Show.new(sequence: sequence) + expect(instance.sequence_version).to eq(sequence.sequence_version) + end + end + + describe "#sequence_publication" do + it "returns the associated sequence_publication if association is loaded" do + allow(sequence.association(:sequence_publication)).to receive(:loaded?).and_return(true) + instance = Sequences::Show.new(sequence: sequence) + expect(instance.sequence_publication).to eq(sequence.sequence_publication) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6a97cf8d5a..1a0e4480ce 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -85,9 +85,7 @@ def clear! config.order = "random" end -FAKE_ATTACHMENT_URL = "https://cdn.shopify.com/s/files/1/2040/0" \ - "289/files/FarmBot.io_Trimmed_Logo_Gray_o" \ - "n_Transparent_1_434x200.png?v=1525220371" +FAKE_ATTACHMENT_URL = "https://example.com/image.jpg" def simulate_fbos_request(version = "17.1.2") ua = "FARMBOTOS/#{version} (RPI3) RPI3 (#{version})"