Add image strategy

CHANGELOG.md

@@ -5,10 +5,13 @@ All notable changes to this project will be documented in this file.
 ### Enhancements
 - Update documentation about `start_recording_screen`
 - Port `send_keys/type` for active element
+- Support `find_element/s :image, partial_image`
 
 ### Bug fixes
 
 ### Deprecations
+- Deprecate experimental `ImageElement` in favor of `Element`
+    - Anyone does not need to care this
 
 ## [1.8.4] - 2018-07-28
 ### Enhancements

lib/appium_lib_core/common/base/bridge/mjsonwp.rb

@@ -22,70 +22,6 @@ def commands(command)
             ::Appium::Core::Commands::MJSONWP::COMMANDS[command]
           end
 
-          #
-          # @return [::Appium::Core::ImageElement|nil]
-          # @raise [::Selenium::WebDriver::Error::TimeOutError|::Selenium::WebDriver::Error::WebDriverError]
-          #
-          def find_element_by_image(full_image:, partial_image:, match_threshold: nil, visualize: false)
-            options = {}
-            options[:threshold] = match_threshold unless match_threshold.nil?
-            options[:visualize] = visualize
-
-            params = {}
-            params[:mode] = :matchTemplate
-            params[:firstImage] = full_image
-            params[:secondImage] = partial_image
-            params[:options] = options if options
-
-            result = execute(:compare_images, {}, params)
-            rect = result['rect']
-
-            if rect
-              return ::Appium::Core::ImageElement.new(self,
-                                                      rect['x'],
-                                                      rect['y'],
-                                                      rect['width'],
-                                                      rect['height'],
-                                                      result['visualization'])
-            end
-            nil
-          end
-
-          #
-          # @return [[]|[::Appium::Core::ImageElement]]
-          # @raise [::Selenium::WebDriver::Error::TimeOutError|::Selenium::WebDriver::Error::WebDriverError]
-          #
-          def find_elements_by_image(full_image:, partial_images:, match_threshold: nil, visualize: false)
-            options = {}
-            options[:threshold] = match_threshold unless match_threshold.nil?
-            options[:visualize] = visualize
-
-            params = {}
-            params[:mode] = :matchTemplate
-            params[:firstImage] = full_image
-            params[:options] = options if options
-
-            partial_images.each_with_object([]) do |partial_image, acc|
-              params[:secondImage] = partial_image
-
-              begin
-                result = execute(:compare_images, {}, params)
-                rect = result['rect']
-
-                if result['rect']
-                  acc.push ::Appium::Core::ImageElement.new(self,
-                                                            rect['x'],
-                                                            rect['y'],
-                                                            rect['width'],
-                                                            rect['height'],
-                                                            result['visualization'])
-                end
-              rescue ::Selenium::WebDriver::Error::WebDriverError => e
-                acc if e.message.include?('Cannot find any occurrences')
-              end
-            end
-          end
-
           def take_element_screenshot(element)
             execute :take_element_screenshot, id: element.ref
           end

lib/appium_lib_core/common/base/bridge/w3c.rb

@@ -1,5 +1,3 @@
-require 'base64'
-
 module Appium
   module Core
     class Base
@@ -115,70 +113,6 @@ def find_elements_by(how, what, parent = nil)
             ids.map { |id| ::Selenium::WebDriver::Element.new self, element_id_from(id) }
           end
 
-          #
-          # @return [::Appium::Core::ImageElement|nil]
-          # @raise [::Selenium::WebDriver::Error::TimeOutError|::Selenium::WebDriver::Error::WebDriverError]
-          #
-          def find_element_by_image(full_image:, partial_image:, match_threshold: nil, visualize: false)
-            options = {}
-            options[:threshold] = match_threshold unless match_threshold.nil?
-            options[:visualize] = visualize
-
-            params = {}
-            params[:mode] = :matchTemplate
-            params[:firstImage] = full_image
-            params[:secondImage] = partial_image
-            params[:options] = options if options
-
-            result = execute(:compare_images, {}, params)
-            rect = result['rect']
-
-            if rect
-              return ::Appium::Core::ImageElement.new(self,
-                                                      rect['x'],
-                                                      rect['y'],
-                                                      rect['width'],
-                                                      rect['height'],
-                                                      result['visualization'])
-            end
-            nil
-          end
-
-          #
-          # @return [[]|[::Appium::Core::ImageElement]]
-          # @raise [::Selenium::WebDriver::Error::TimeOutError|::Selenium::WebDriver::Error::WebDriverError]
-          #
-          def find_elements_by_image(full_image:, partial_images:, match_threshold: nil, visualize: false)
-            options = {}
-            options[:threshold] = match_threshold unless match_threshold.nil?
-            options[:visualize] = visualize
-
-            params = {}
-            params[:mode] = :matchTemplate
-            params[:firstImage] = full_image
-            params[:options] = options if options
-
-            partial_images.each_with_object([]) do |partial_image, acc|
-              params[:secondImage] = partial_image
-
-              begin
-                result = execute(:compare_images, {}, params)
-                rect = result['rect']
-
-                if result['rect']
-                  acc.push ::Appium::Core::ImageElement.new(self,
-                                                            rect['x'],
-                                                            rect['y'],
-                                                            rect['width'],
-                                                            rect['height'],
-                                                            result['visualization'])
-                end
-              rescue ::Selenium::WebDriver::Error::WebDriverError => e
-                acc if e.message.include?('Cannot find any occurrences')
-              end
-            end
-          end
-
           # For Appium
           # override
           # called in `extend DriverExtensions::HasNetworkConnection`

lib/appium_lib_core/common/base/driver.rb

@@ -798,73 +798,47 @@ def compare_images(mode: :matchFeatures, first_image:, second_image:, options: n
           @bridge.compare_images(mode: mode, first_image: first_image, second_image: second_image, options: options)
         end
 
-        # Return ImageElement if current view has a partial image
+        # @since Appium 1.8.2
+        # Return an element if current view has a partial image. The logic depends on template matching by OpenCV.
+        # @see https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/image-comparison.md
+        # You can handle settings for the comparision following below.
+        # @see https://github.com/appium/appium-base-driver/blob/master/lib/basedriver/device-settings.js#L6
         #
         # @param [String] png_img_path A path to a partial image you'd like to find
-        # @param [Flood] match_threshold At what normalized threshold to reject
-        # @param [Bool] visualize Makes the endpoint to return an image, which contains the visualized result of
-        #                         the corresponding picture matching operation. This option is disabled by default.
         #
-        # @return [::Appium::Core::ImageElement]
-        # @raise [::Appium::Core::Error::NoSuchElementError|::Appium::Core::Error::CoreError] No such element
+        # @return [::Selenium::WebDriver::Element]
         #
         # @example
         #
+        #     @@driver.update_settings({ fixImageFindScreenshotDims: false, fixImageTemplateSize: true,
+        #                                autoUpdateImageElementPosition: true })
         #     e = @@driver.find_element_by_image './test/functional/data/test_element_image.png'
         #
-        def find_element_by_image(png_img_path, match_threshold: DEFAULT_MATCH_THRESHOLD, visualize: false)
-          full_image = @bridge.screenshot
-          partial_image = Base64.encode64 File.read(png_img_path)
-
-          element = begin
-            @bridge.find_element_by_image(full_image: full_image,
-                                          partial_image: partial_image,
-                                          match_threshold: match_threshold,
-                                          visualize: visualize)
-          rescue Selenium::WebDriver::Error::TimeOutError
-            raise ::Appium::Core::Error::NoSuchElementError
-          rescue ::Selenium::WebDriver::Error::WebDriverError => e
-            raise ::Appium::Core::Error::NoSuchElementError if e.message.include?('Cannot find any occurrences')
-            raise ::Appium::Core::Error::CoreError, e.message
-          end
-          raise ::Appium::Core::Error::NoSuchElementError if element.nil?
-
-          element
+        def find_element_by_image(png_img_path)
+          template = Base64.encode64 File.read png_img_path
+          find_element :image, template
         end
 
-        # Return ImageElement if current view has partial images
+        # @since Appium 1.8.2
+        # Return elements if current view has a partial image. The logic depends on template matching by OpenCV.
+        # @see https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/image-comparison.md
+        # You can handle settings for the comparision following below.
+        # @see https://github.com/appium/appium-base-driver/blob/master/lib/basedriver/device-settings.js#L6
         #
-        # @param [[String]] png_img_paths Paths to a partial image you'd like to find
-        # @param [Flood] match_threshold At what normalized threshold to reject
-        # @param [Bool] visualize Makes the endpoint to return an image, which contains the visualized result of
-        #                         the corresponding picture matching operation. This option is disabled by default.
+        # @param [String] png_img_path A path to a partial image you'd like to find
         #
-        # @return [[::Appium::Core::ImageElement]]
-        # @return [::Appium::Core::Error::CoreError]
+        # @return [::Selenium::WebDriver::Element]
         #
         # @example
         #
+        #     @@driver.update_settings({ fixImageFindScreenshotDims: false, fixImageTemplateSize: true,
+        #                                autoUpdateImageElementPosition: true })
         #     e = @@driver.find_elements_by_image ['./test/functional/data/test_element_image.png']
         #     e == [] # if the `e` is empty
         #
-        def find_elements_by_image(png_img_paths, match_threshold: DEFAULT_MATCH_THRESHOLD, visualize: false)
-          full_image = @bridge.screenshot
-
-          partial_images = png_img_paths.map do |png_img_path|
-            Base64.encode64 File.read(png_img_path)
-          end
-
-          begin
-            @bridge.find_elements_by_image(full_image: full_image,
-                                           partial_images: partial_images,
-                                           match_threshold: match_threshold,
-                                           visualize: visualize)
-          rescue Selenium::WebDriver::Error::TimeOutError
-            []
-          rescue ::Selenium::WebDriver::Error::WebDriverError => e
-            return [] if e.message.include?('Cannot find any occurrences')
-            raise ::Appium::Core::Error::CoreError, e.message
-          end
+        def find_elements_by_image(png_img_path)
+          template = Base64.encode64 File.read png_img_path
+          find_elements :image, template
         end
       end # class Driver
     end # class Base

lib/appium_lib_core/common/base/search_context.rb

@@ -6,6 +6,7 @@ module SearchContext
 
         FINDERS = ::Selenium::WebDriver::SearchContext::FINDERS.merge(
           accessibility_id: 'accessibility id',
+          image: '-image',
           # Android
           uiautomator: '-android uiautomator',
           # iOS
@@ -25,6 +26,12 @@ module SearchContext
         # iOS can find with a [UIAutomation command](https://developer.apple.com/library/ios/documentation/ToolsLanguages/Reference/UIAWindowClassReference/UIAWindow/UIAWindow.html#//apple_ref/doc/uid/TP40009930).
         # iOS, only for XCUITest(WebDriverAgent), can find with a [class chain]( https://github.com/facebook/WebDriverAgent/wiki/Queries)
         #
+        # Find with image.
+        # Return an element if current view has a partial image. The logic depends on template matching by OpenCV.
+        # @see https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/image-comparison.md
+        # You can handle settings for the comparision following below.
+        # @see https://github.com/appium/appium-base-driver/blob/master/lib/basedriver/device-settings.js#L6
+        #
         # @overload find_element(how, what)
         #   @param [Symbol, String] how The method to find the element by
         #   @param [String] what The locator to use
@@ -35,10 +42,15 @@ module SearchContext
         # @return [Element]
         # @raise [Error::NoSuchElementError] if the element doesn't exist
         #
-        # @example Find element with accessibility id
+        # @example Find element with each keys
+        #
+        #     # with accessibility id. All platforms.
         #     find_elements :accessibility_id, 'Animation'
         #     find_elements :accessibility_id, 'Animation'
         #
+        #     # with base64 encoded template image. All platforms.
+        #     find_elements :image, Base64.encode64(File.read(file_path))
+        #
         #     # For Android
         #     ## With uiautomator
         #     find_elements :uiautomator, 'new UiSelector().clickable(true)'
@@ -100,8 +112,7 @@ def find_elements(*args)
         private
 
         def _set_by_from_finders(how)
-          finders = FINDERS
-          by = finders[how.to_sym]
+          by = FINDERS[how.to_sym]
           raise ArgumentError, "cannot find element by #{how.inspect}" unless by
           by
         end

lib/appium_lib_core/device.rb

@@ -1,7 +1,3 @@
-require_relative 'element/image'
-
-require 'base64'
-
 module Appium
   module Core
     module Device