diff --git a/api/README.md b/api/README.md index bd201fe..b39ea9d 100644 --- a/api/README.md +++ b/api/README.md @@ -31,4 +31,5 @@ $ FIELDAGENT_SERVER=https://apistaging.sentera.com ruby upsert_feature_set.rb | Ruby | `$ ruby import_files.rb` | `FIELDAGENT_ACCESS_TOKEN=PAmnCNUyosKShN9K1AEflLOw6T7bA2fRTWTg-vL3P5Y FIELDAGENT_SERVER=https://api.sentera.com FIELD_SENTERA_ID=agwmnou_AS_lk07AcmeOrg_CV_deve_773b47acb_240514_160730 FILE_PATH="../test_files/test.geojson" CONTENT_TYPE="application/json" ruby import_files.rb` | | Ruby | `$ ruby upsert_feature_set.rb` | `FIELDAGENT_ACCESS_TOKEN=PAmnCNUyosKShN9K1AEflLOw6T7bA2fRTWTg-vL3P5Y FIELDAGENT_SERVER=https://api.sentera.com SURVEY_SENTERA_ID=mjlmmrw_CO_lk07AcmeOrg_CV_deve_773b47acb_240514_160730 GEOMETRY_PATH="../test_files/test.geojson" FILES_PATH="../test_files" FILE_EXT="*.jpeg" ruby upsert_feature_set.rb` | | Ruby | `$ ruby upsert_files.rb` | `FIELDAGENT_ACCESS_TOKEN=PAmnCNUyosKShN9K1AEflLOw6T7bA2fRTWTg-vL3P5Y FIELDAGENT_SERVER=https://api.sentera.com FILE_PATH="../test_files/test.geojson" CONTENT_TYPE="application/json" FIELD_SENTERA_ID=agwmnou_AS_lk07AcmeOrg_CV_deve_773b47acb_240514_160730 ORGANIZATION_SENTERA_ID="jiqn6qi_OR_5qytAcmeOrg_CV_deve_0f569249e_250206_162717" ruby upsert_files.rb` | +| Ruby | `$ ruby upsert_images.rb` | `FIELDAGENT_ACCESS_TOKEN=PAmnCNUyosKShN9K1AEflLOw6T7bA2fRTWTg-vL3P5Y FIELDAGENT_SERVER=https://api.sentera.com IMAGES_PATH="../test_files" SURVEY_SENTERA_ID=mjlmmrw_CO_lk07AcmeOrg_CV_deve_773b47acb_240514_160730 FILE_EXT="*.jpeg" SENSOR_TYPE="RGB" ruby upsert_images.rb` | | Ruby | `$ ruby upsert_mosaics.rb` | `FIELDAGENT_ACCESS_TOKEN=PAmnCNUyosKShN9K1AEflLOw6T7bA2fRTWTg-vL3P5Y FIELDAGENT_SERVER=https://api.sentera.com FILE_PATH="../test_files/test.tif" SURVEY_SENTERA_ID=mjlmmrw_CO_lk07AcmeOrg_CV_deve_773b47acb_240514_160730 ruby upsert_mosaics.rb` | diff --git a/api/import_feature_set.rb b/api/import_feature_set.rb index 2b9ac88..e4735de 100755 --- a/api/import_feature_set.rb +++ b/api/import_feature_set.rb @@ -17,6 +17,7 @@ require 'json' require 'digest' require '../utils/parallel' +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -138,45 +139,6 @@ def create_file_uploads(file_paths, survey_sentera_id, feature_set_sentera_id) json.dig('data', 'create_file_uploads') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Array[Object]] file_uploads FileUpload GraphQL -# objects created by the -# create_file_uploads mutation -# @param [Array[string]] file_paths Array of paths to -# the files to upload -# -# @return [void] -# -def upload_files(file_uploads, file_paths) - puts 'Upload files' - - file_uploads_map = {} - file_uploads.each.with_index do |file_upload, index| - file_path = file_paths[index] - file_uploads_map[file_path] = file_upload - end - - Parallel.each(file_paths, in_threads: 6) do |file_path| - file_upload = file_uploads_map[file_path] - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end - end -end - # # This method demonstrates how to use the IDs of the files that # were previously uploaded to Sentera's cloud storage with the diff --git a/api/import_feature_set_legacy.rb b/api/import_feature_set_legacy.rb index 2d08659..b4352d1 100755 --- a/api/import_feature_set_legacy.rb +++ b/api/import_feature_set_legacy.rb @@ -17,6 +17,7 @@ require 'json' require 'digest' require '../utils/parallel' +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -116,45 +117,6 @@ def create_file_uploads(file_paths) json.dig('data', 'create_file_uploads') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Array[Object]] file_uploads FileUpload GraphQL -# objects created by the -# create_file_uploads mutation -# @param [Array[string]] file_paths Array of paths to -# the files to upload -# -# @return [void] -# -def upload_files(file_uploads, file_paths) - puts 'Upload files' - - file_uploads_map = {} - file_uploads.each.with_index do |file_upload, index| - file_path = file_paths[index] - file_uploads_map[file_path] = file_upload - end - - Parallel.each(file_paths, in_threads: 6) do |file_path| - file_upload = file_uploads_map[file_path] - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end - end -end - # # This method demonstrates how to use the IDs of the files that # were previously uploaded to Sentera's cloud storage with the diff --git a/api/import_files.rb b/api/import_files.rb index a50f7fd..296753a 100644 --- a/api/import_files.rb +++ b/api/import_files.rb @@ -16,7 +16,7 @@ require 'net/http' require 'json' require 'digest' - +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -73,34 +73,6 @@ def create_file_upload(file_path, content_type) json.dig('data', 'create_file_upload') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Object] file_upload FileUpload GraphQL -# object created by the -# create_file_upload mutation -# @param [string] file_path Path of the file to upload -# -# @return [void] -# -def upload_file(file_upload, file_path) - puts 'Upload file' - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end -end - # # This method demonstrates how to use the ID of the file that # was previously uploaded to Sentera's cloud storage with the diff --git a/api/upsert_feature_set.rb b/api/upsert_feature_set.rb index bb6e545..3b571ab 100755 --- a/api/upsert_feature_set.rb +++ b/api/upsert_feature_set.rb @@ -17,6 +17,7 @@ require 'json' require 'digest' require '../utils/parallel' +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -130,46 +131,6 @@ def create_file_uploads(file_paths, survey_sentera_id) json.dig('data', 'create_file_uploads') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Array[Object]] file_uploads FileUpload GraphQL -# objects created by the -# create_file_uploads mutation -# @param [Array[string]] file_paths Array of paths to -# the files to upload -# -# @return [void] -# -def upload_files(file_uploads, file_paths) - puts 'Upload files' - - file_uploads_map = file_uploads.each_with_object({}) do |file_upload, map| - s3_url = file_upload['s3_url'] - filename = File.basename(s3_url) - map[filename] = file_upload - end - - Parallel.each(file_paths, in_threads: 6) do |file_path| - filename = File.basename(file_path) - file_upload = file_uploads_map[filename] - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end - end -end - # # This method demonstrates how to use the IDs of the files that # were previously uploaded to Sentera's cloud storage with the diff --git a/api/upsert_files.rb b/api/upsert_files.rb index 63051dd..164bca3 100644 --- a/api/upsert_files.rb +++ b/api/upsert_files.rb @@ -16,7 +16,7 @@ require 'net/http' require 'json' require 'digest' - +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -87,34 +87,6 @@ def create_file_upload(file_path, content_type, field_sentera_id, organization_s json.dig('data', 'create_file_upload') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Object] file_upload FileUpload GraphQL -# object created by the -# create_file_upload mutation -# @param [string] file_path Path of the file to upload -# -# @return [void] -# -def upload_file(file_upload, file_path) - puts 'Upload file' - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end -end - # # This method demonstrates how to attach the previously uploaded # file to a field by using the upsert_files GraphQL mutation. diff --git a/api/upsert_images.rb b/api/upsert_images.rb new file mode 100644 index 0000000..ce695b3 --- /dev/null +++ b/api/upsert_images.rb @@ -0,0 +1,202 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +# ================================================================== +# A Ruby example that demonstrates the workflow for uploading images +# to Sentera's FieldAgent platform using the create_file_uploads +# and upsert_images GraphQL mutations. +# +# Contact devops@sentera.com with any questions. +# ================================================================== + +require '../utils/utils' +verify_ruby_version + +require 'net/http' +require 'json' +require 'digest' +require '../utils/parallel' +require '../utils/upload' + +# If you want to debug this script, run the following gem install +# commands. Then uncomment the require statements below, and put +# debugger statements in the code to trace the code execution. +# +# > gem install pry +# > gem install pry-byebug +# +# require 'pry' +# require 'pry-byebug' + +# +# This method demonstrates how to use the create_image_uploads +# mutation in Sentera's GraphQL API to prepare images for +# upload to Sentera's cloud storage. +# +# @param [Array[Hash]] image_props Array of image properties +# @param [string] survey_sentera_id Sentera ID of the survey +# within FieldAgent that will +# be the parent of the images. +# @param [string] sensor_type The type of sensor that captured the images +# +# @return [Hash] Hash containing results of the GraphQL request +# +def create_image_uploads(image_props, survey_sentera_id, sensor_type) + puts 'Create image uploads' + + images = image_props.map do |props| + { + filename: props[:filename], + byte_size: props[:byte_size], + checksum: props[:checksum], + content_type: props[:content_type], + sensor_type: sensor_type + } + end + + gql = <<~GQL + mutation CreateImageUploads( + $survey_sentera_id: ID!, + $images: [ImageUploadInput!]! + ) { + create_image_uploads( + survey_sentera_id: $survey_sentera_id + images: $images + ) { + id + headers + s3_url + upload_url + } + } + GQL + + variables = { + survey_sentera_id: survey_sentera_id, + images: images + } + + response = make_graphql_request(gql, variables) + json = JSON.parse(response.body) + json.dig('data', 'create_image_uploads') +end + +# +# This method demonstrates how to use the IDs of the images that +# were previously uploaded to Sentera's cloud storage with the +# upsert_images GraphQL mutation. +# +# @param [string] survey_sentera_id Sentera ID of the survey +# within FieldAgent to create +# the images under +# @param [Array[Object]] image_uploads ImageUpload GraphQL +# objects created by the +# create_image_uploads mutation +# @param [string] sensor_type The type of sensor that captured the images +# @param [Array[Hash]] image_props Array of image properties +# +# @return [Hash] Hash containing results of the GraphQL request +# +def upsert_images(survey_sentera_id, image_uploads, sensor_type, image_props) + puts 'Upsert images' + + gql = <<~GQL + mutation UpsertImages( + $survey_sentera_id: ID! + $images: [ImageImport!]! + ) { + upsert_images( + survey_sentera_id: $survey_sentera_id + images: $images + ) { + succeeded { + ... on Image { + sentera_id + filename + } + } + failed { + attributes { + key + details + attribute + } + } + } + } + GQL + + variables = { + survey_sentera_id: survey_sentera_id, + images: image_uploads.map do |image_upload| + filename = File.basename(image_upload['s3_url']) + props = image_props.find { |p| p[:filename] == filename } + + { + # The following attributes are required when creating images + # using the upsert_images mutation. Adjust as needed. + # Note the use of the file_key attribute to reference + # the previously uploaded image. + altitude: 0, + calculated_index: 'UNKNOWN', + captured_at: Time.now.utc.iso8601, + color_applied: 'UNKNOWN', + filename: filename, + key: image_upload['id'], + gps_carrier_phase_status: 'STANDARD', + gps_horizontal_accuracy: 0, + gps_vertical_accuracy: 0, + latitude: 0, + longitude: 0, + sensor_type: sensor_type, + size: props[:byte_size] + } + end + } + + response = make_graphql_request(gql, variables) + json = JSON.parse(response.body) + json.dig('data', 'upsert_images') +end + +# MAIN + +# ************************************************** +# Set these variables based on the images you want +# to upload and the survey within FieldAgent to +# create the images under. +images_path = ENV.fetch('IMAGES_PATH', '.') # Your fully qualified path to a folder containing the images to upload +file_ext = ENV.fetch('FILE_EXT', '*.*') # Your image file extension +sensor_type = ENV.fetch('SENSOR_TYPE', 'UNKNOWN') # Your sensor type for the images being uploaded +survey_sentera_id = ENV.fetch('SURVEY_SENTERA_ID', nil) # Your existing survey Sentera ID +# ************************************************** + +unless File.exist?(images_path) + raise "IMAGES_PATH #{images_path} does not exist" +end + +if survey_sentera_id.nil? + raise 'SURVEY_SENTERA_ID environment variable must be specified' +end + +# Step 1: Create image uploads for the images +image_props = read_file_props(images_path, file_ext) +image_uploads = create_image_uploads(image_props, survey_sentera_id, sensor_type) +if image_uploads.nil? + puts 'Failed' + exit +end + +# Step 2: Upload the images +image_paths = image_props.map { |props| props[:path] } +upload_files(image_uploads, image_paths) + +# Step 3: Create images in FieldAgent using the uploaded images +results = upsert_images(survey_sentera_id, image_uploads, sensor_type, image_props) + +if results && results['succeeded'].any? + puts "Done! Images for #{survey_sentera_id} were created in FieldAgent." +else + puts "Failed due to error: #{results['failed'].inspect}" +end diff --git a/api/upsert_mosaics.rb b/api/upsert_mosaics.rb index f8e1e8d..bfc090e 100644 --- a/api/upsert_mosaics.rb +++ b/api/upsert_mosaics.rb @@ -17,6 +17,7 @@ require 'json' require 'digest' require 'time' +require '../utils/upload' # If you want to debug this script, run the following gem install # commands. Then uncomment the require statements below, and put @@ -82,34 +83,6 @@ def create_file_upload(file_path, survey_sentera_id) json.dig('data', 'create_file_upload') end -# -# This method demonstrates how to upload a file to -# Sentera's cloud storage using the URL and headers -# that were retrieved via the create_file_upload -# GraphQL mutation. -# -# @param [Object] file_upload FileUpload GraphQL -# object created by the -# create_file_upload mutation -# @param [string] file_path Path of the file to upload -# -# @return [void] -# -def upload_file(file_upload, file_path) - puts 'Upload file' - - uri = URI(file_upload['upload_url']) - file_contents = File.read(file_path) - Net::HTTP.start(uri.host) do |http| - puts "Upload #{file_path} to S3" - response = http.send_request('PUT', - uri, - file_contents, - file_upload['headers']) - puts "Done uploading #{file_path}, response.code = #{response.code}" - end -end - # # This method demonstrates how to use the ID of the file that # was previously uploaded to Sentera's cloud storage with the diff --git a/test_files/test.jpeg b/test_files/test.jpeg index ba41fff..3ddcf0a 100644 Binary files a/test_files/test.jpeg and b/test_files/test.jpeg differ diff --git a/test_files/test2.jpeg b/test_files/test2.jpeg new file mode 100644 index 0000000..34f06dc Binary files /dev/null and b/test_files/test2.jpeg differ diff --git a/test_files/test3.jpeg b/test_files/test3.jpeg new file mode 100644 index 0000000..1788ecf Binary files /dev/null and b/test_files/test3.jpeg differ diff --git a/utils/upload.rb b/utils/upload.rb new file mode 100644 index 0000000..abf12c6 --- /dev/null +++ b/utils/upload.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +# MIME content type mapping for common file extensions +CONTENT_TYPES = { + '.geojson' => 'application/geo+json', + '.json' => 'application/json', + '.tif' => 'image/tiff', + '.tiff' => 'image/tiff', + '.jpg' => 'image/jpeg', + '.jpeg' => 'image/jpeg', + '.png' => 'image/png', + '.pdf' => 'application/pdf', + '.zip' => 'application/zip', + '.txt' => 'text/plain', + '.csv' => 'text/csv', + '.xml' => 'application/xml', + '.shp' => 'application/octet-stream', + '.kml' => 'application/vnd.google-earth.kml+xml', + '.kmz' => 'application/vnd.google-earth.kmz' +}.freeze + +# +# Reads the files at a path for a specified extension +# +# @param [string] files_path Path to a directory containing +# the files +# @param [string] file_ext Extension of files to return +# +# @return [Array[string]] Array of file paths +# +def read_file_paths(files_path, file_ext) + raise "Files path #{files_path} does not exist" unless Dir.exist?(files_path) + + files_path += '/' unless files_path.end_with?('/') + Dir.glob("#{files_path}#{file_ext}") +end + +# +# Reads file properties for files at a path with a specified extension. +# Returns an array of hashes containing file metadata including path, filename, +# byte size, MD5 checksum, and automatically detected content type based on +# file extension. +# +# @param [string] files_path Path to a directory containing the files +# @param [string] file_ext Extension of files to return +# +# @return [Array[Hash]] Array of hashes with keys: :path, :filename, :byte_size, +# :checksum, :content_type +# +def read_file_props(files_path, file_ext) + raise "Files path #{files_path} does not exist" unless Dir.exist?(files_path) + + read_file_paths(files_path, file_ext).map do |file_path| + ext = File.extname(file_path).downcase + content_type = CONTENT_TYPES[ext] || 'application/octet-stream' + + { + path: file_path, + filename: File.basename(file_path), + byte_size: File.size(file_path), + checksum: Digest::MD5.base64digest(File.read(file_path)), + content_type: content_type + } + end +end + +# +# This method demonstrates how to upload a file to +# Sentera's cloud storage using the URL and headers +# that were retrieved via the create_file_upload +# GraphQL mutation. +# +# @param [Object] file_upload FileUpload GraphQL +# object created by the +# create_file_upload mutation +# @param [string] file_path Path of the file to upload +# +# @return [void] +# +def upload_file(file_upload, file_path) + puts 'Upload file' + + uri = URI(file_upload['upload_url']) + file_contents = File.read(file_path) + Net::HTTP.start(uri.host) do |http| + puts "Upload #{file_path} to S3" + response = http.send_request('PUT', + uri, + file_contents, + file_upload['headers']) + puts "Done uploading #{file_path}, response.code = #{response.code}" + end +end + +# +# This method demonstrates how to upload files to +# Sentera's cloud storage using the URL and headers +# that were retrieved via the create_file_upload, +# create_file_uploads or create_image_uploads +# GraphQL mutations +# +# @param [Array[Object]] file_uploads FileUpload GraphQL +# objects created by the +# create_file_uploads mutation +# @param [Array[string]] file_paths Array of paths to +# the files to upload +# +# @return [void] +# +def upload_files(file_uploads, file_paths) + puts 'Upload files' + + file_uploads_map = {} + file_uploads.each.with_index do |file_upload, index| + file_path = file_paths[index] + file_uploads_map[file_path] = file_upload + end + + Parallel.each(file_paths, in_threads: 6) do |file_path| + file_upload = file_uploads_map[file_path] + + uri = URI(file_upload['upload_url']) + file_contents = File.read(file_path) + Net::HTTP.start(uri.host) do |http| + puts "Upload #{file_path} to S3" + response = http.send_request('PUT', + uri, + file_contents, + file_upload['headers']) + puts "Done uploading #{file_path}, response.code = #{response.code}" + end + end +end diff --git a/utils/utils.rb b/utils/utils.rb index a717b29..76abfaf 100644 --- a/utils/utils.rb +++ b/utils/utils.rb @@ -41,29 +41,19 @@ def make_graphql_request(gql, variables = {}) request = Net::HTTP::Post.new(uri.path, headers) request.body = { query: gql, variables: variables }.to_json - puts "Make GraphQL request: uri = #{GQL_ENDPOINT}, gql = #{gql}, variables = #{variables}" + puts "Make GraphQL request to uri = #{GQL_ENDPOINT}" + puts gql + puts "variables = #{variables}" + response = http.request(request) + + puts "" puts "GraphQL response: code = #{response.code}, body = #{response.body}" + puts "" response end -# -# Reads the files at a path for a specified extension -# -# @param [string] files_path Path to a directory containing -# the files -# @param [string] file_ext Extension of files to return -# -# @return [Array[string]] Array of file paths -# -def read_file_paths(files_path, file_ext) - raise "Files path #{files_path} does not exist" unless Dir.exist?(files_path) - - files_path += '/' unless files_path.end_with?('/') - Dir.glob("#{files_path}#{file_ext}") -end - GQL_ENDPOINT = "#{ENV.fetch('FIELDAGENT_SERVER', 'https://api.sentera.com')}/graphql" # Defaults to FieldAgent production FIELDAGENT_ACCESS_TOKEN_FILENAME = 'fieldagent_access_token.txt'