add find by image commands and tests

.gitignore

@@ -18,3 +18,4 @@ dist
 .cache
 __pycache__
 .idea
+.pytest_cache

appium/webdriver/common/mobileby.py

@@ -21,3 +21,4 @@ class MobileBy(By):
     IOS_CLASS_CHAIN = '-ios class chain'
     ANDROID_UIAUTOMATOR = '-android uiautomator'
     ACCESSIBILITY_ID = 'accessibility id'
+    IMAGE = '-image'

appium/webdriver/imagelement.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+
+class ImageElement:
+
+    def __init__(self, driver, x, y, width, height):
+        self.driver = driver
+        self.center_x = math.floor(x + width / 2)
+        self.center_y = math.floor(y + height / 2)
+        self.x = x
+        self.y = y
+        self.width = width
+        self.height = height
+
+    def click(self):
+        """
+        Clicks in the middle of an image bounds
+        """
+        return self.driver.tap([(self.center_x, self.center_y)])
+
+    @property
+    def size(self):
+        return {'width': self.width, 'height': self.height}
+
+    @property
+    def location(self):
+        return {'x': self.x, 'y': self.y}
+
+    @property
+    def rect(self):
+        return {
+            'width': self.width,
+            'height': self.height,
+            'x': self.x,
+            'y': self.y
+        }
+
+    def is_displayed(self):
+        return True

appium/webdriver/webdriver.py

@@ -18,6 +18,7 @@
 from .errorhandler import MobileErrorHandler
 from .switch_to import MobileSwitchTo
 from .webelement import WebElement as MobileWebElement
+from .imagelement import ImageElement
 
 from appium.webdriver.clipboard_content_type import ClipboardContentType
 from appium.webdriver.common.mobileby import MobileBy
@@ -26,13 +27,16 @@
 
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
-from selenium.common.exceptions import TimeoutException, InvalidArgumentException
+from selenium.common.exceptions import (TimeoutException,
+        WebDriverException, InvalidArgumentException, NoSuchElementException)
 
 from selenium.webdriver.remote.command import Command as RemoteCommand
 
 import base64
 import copy
 
+DEFAULT_MATCH_THRESHOLD = 0.5
+
 # From remote/webdriver.py
 _W3C_CAPABILITY_NAMES = frozenset([
     'acceptInsecureCerts',
@@ -84,7 +88,9 @@ def _make_w3c_caps(caps):
             always_match['moz:firefoxOptions'] = new_opts
     return {'firstMatch': [{}], 'alwaysMatch': always_match}
 
+
 class WebDriver(webdriver.Remote):
+
     def __init__(self, command_executor='http://127.0.0.1:4444/wd/hub',
                  desired_capabilities=None, browser_profile=None, proxy=None, keep_alive=False):
 
@@ -102,6 +108,7 @@ def __init__(self, command_executor='http://127.0.0.1:4444/wd/hub',
         By.IOS_CLASS_CHAIN = MobileBy.IOS_CLASS_CHAIN
         By.ANDROID_UIAUTOMATOR = MobileBy.ANDROID_UIAUTOMATOR
         By.ACCESSIBILITY_ID = MobileBy.ACCESSIBILITY_ID
+        By.IMAGE = MobileBy.IMAGE
 
     def start_session(self, capabilities, browser_profile=None):
         """
@@ -206,6 +213,9 @@ def find_element(self, by=By.ID, value=None):
             # elif by == By.NAME:
             #     by = By.CSS_SELECTOR
             #     value = '[name="%s"]' % value
+        if by == By.IMAGE:
+            return self.find_element_by_image(value)
+
         return self.execute(RemoteCommand.FIND_ELEMENT, {
             'using': by,
             'value': value})['value']
@@ -235,6 +245,10 @@ def find_elements(self, by=By.ID, value=None):
 
         # Return empty list if driver returns null
         # See https://github.com/SeleniumHQ/selenium/issues/4555
+
+        if by == By.IMAGE:
+            return self.find_elements_by_image(value)
+
         return self.execute(RemoteCommand.FIND_ELEMENTS, {
             'using': by,
             'value': value})['value'] or []
@@ -327,6 +341,52 @@ def find_elements_by_android_uiautomator(self, uia_string):
         """
         return self.find_elements(by=By.ANDROID_UIAUTOMATOR, value=uia_string)
 
+    def find_element_by_image(self, png_img_path,
+                              match_threshold=DEFAULT_MATCH_THRESHOLD):
+        """Finds a portion of a screenshot by an image.
+        Uses driver.find_image_occurrence under the hood.
+
+        :Args:
+        - png_img_path - a string corresponding to the path of a PNG image
+        - match_threshold - a double between 0 and 1 below which matches will
+          be rejected as element not found
+
+        :return: an ImageElement object
+        """
+        screenshot = self.get_screenshot_as_base64()
+        with open(png_img_path, 'rb') as png_file:
+            b64_data = base64.encodestring(png_file.read())
+        try:
+            res = self.find_image_occurrence(screenshot, b64_data,
+                                             threshold=match_threshold)
+        except WebDriverException as e:
+            if 'Cannot find any occurrences' in str(e):
+                raise NoSuchElementException(e)
+            raise
+        rect = res['rect']
+        return ImageElement(self, rect['x'], rect['y'], rect['width'],
+                            rect['height'])
+
+    def find_elements_by_image(self, png_img_path,
+                               match_threshold=DEFAULT_MATCH_THRESHOLD):
+        """Finds a portion of a screenshot by an image.
+        Uses driver.find_image_occurrence under the hood. Note that this will
+        only ever return at most one element
+
+        :Args:
+        - png_img_path - a string corresponding to the path of a PNG image
+        - match_threshold - a double between 0 and 1 below which matches will
+          be rejected as element not found
+
+        :return: possibly-empty list of ImageElements
+        """
+        els = []
+        try:
+            els.append(self.find_element_by_image(png_img_path, match_threshold))
+        except NoSuchElementException:
+            pass
+        return els
+
     def find_element_by_accessibility_id(self, id):
         """Finds an element by accessibility id.
 
@@ -1419,4 +1479,4 @@ def _addCommands(self):
         self.command_executor._commands[Command.GET_CLIPBOARD] = \
             ('POST', '/session/$sessionId/appium/device/get_clipboard')
         self.command_executor._commands[Command.COMPARE_IMAGES] = \
-            ('POST', '/session/$sessionId/appium/compare_images')
+            ('POST', '/session/$sessionId/appium/compare_images')

test/functional/android/desired_capabilities.py

@@ -14,16 +14,17 @@
 
 import os
 
+
 # Returns abs path relative to this file and not cwd
-PATH = lambda p: os.path.abspath(
-    os.path.join(os.path.dirname(__file__), p)
-)
+def PATH(p):
+    return os.path.abspath(
+        os.path.join(os.path.dirname(__file__), p)
+    )
 
 
 def get_desired_capabilities(app):
     desired_caps = {
         'platformName': 'Android',
-        'platformVersion': '4.2',
         'deviceName': 'Android Emulator',
         'app': PATH('../../apps/' + app),
         'newCommandTimeout': 240

test/functional/android/find_by_image_tests.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from appium import webdriver
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import desired_capabilities
+
+
+class FindByImageTests(unittest.TestCase):
+
+    def setUp(self):
+        desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk')
+        self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
+
+    def tearDown(self):
+        self.driver.quit()
+
+    def test_find_based_on_image_template(self):
+        image_path = desired_capabilities.PATH('find_by_image_success.png')
+        el = WebDriverWait(self.driver, 3).until(
+            EC.presence_of_element_located((By.IMAGE, image_path))
+        )
+        size = el.size
+        self.assertIsNotNone(size['width'])
+        self.assertIsNotNone(size['height'])
+        loc = el.location
+        self.assertIsNotNone(loc['x'])
+        self.assertIsNotNone(loc['y'])
+        rect = el.rect
+        self.assertIsNotNone(rect['width'])
+        self.assertIsNotNone(rect['height'])
+        self.assertIsNotNone(rect['x'])
+        self.assertIsNotNone(rect['y'])
+        self.assertTrue(el.is_displayed())
+        el.click()
+        self.driver.find_element_by_accessibility_id("Alarm")
+
+    def test_find_multiple_elements_by_image_just_returns_one(self):
+        WebDriverWait(self.driver, 3).until(
+            EC.presence_of_element_located((By.ACCESSIBILITY_ID, "App"))
+        )
+        image_path = desired_capabilities.PATH('find_by_image_success.png')
+        els = self.driver.find_elements_by_image(image_path)
+        els[0].click()
+        self.driver.find_element_by_accessibility_id("Alarm")
+
+    def test_find_throws_no_such_element(self):
+        image_path = desired_capabilities.PATH('find_by_image_failure.png')
+        with self.assertRaises(TimeoutException):
+            WebDriverWait(self.driver, 3).until(
+                EC.presence_of_element_located((By.IMAGE, image_path))
+            )
+        with self.assertRaises(NoSuchElementException):
+            self.driver.find_element_by_image(image_path)
+
+
+if __name__ == "__main__":
+    suite = unittest.TestLoader().loadTestsFromTestCase(FindByImageTests)
+    unittest.TextTestRunner(verbosity=2).run(suite)