From 018676ea7096dcbb64c1de6f5d61c24b74471c2c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 19 Jun 2023 23:31:41 -0700 Subject: [PATCH 01/65] add test from hyundai --- .../car/toyota/tests/print_platform_codes.py | 21 +++ selfdrive/car/toyota/tests/test_toyota.py | 140 ++++++++++++++++++ selfdrive/car/toyota/values.py | 35 ++++- 3 files changed, 195 insertions(+), 1 deletion(-) create mode 100755 selfdrive/car/toyota/tests/print_platform_codes.py diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py new file mode 100755 index 00000000000000..1bc8a4e3660e78 --- /dev/null +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +from cereal import car +from selfdrive.car.hyundai.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes + +Ecu = car.CarParams.Ecu +ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} + +if __name__ == "__main__": + for car_model, ecus in FW_VERSIONS.items(): + print() + print(car_model) + for ecu in sorted(ecus, key=lambda x: int(x[0])): + if ecu[0] not in PLATFORM_CODE_ECUS: + continue + + platform_codes = get_platform_codes(ecus[ecu]) + codes = {code for code, _ in platform_codes} + dates = {date for _, date in platform_codes if date is not None} + print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') + print(f' Codes: {codes}') + print(f' Dates: {dates}') diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 5648f75fe7d19a..61553b34bd75f4 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,13 +1,153 @@ #!/usr/bin/env python3 import unittest +from cereal import car +from selfdrive.car.fw_versions import build_fw_dict +# from selfdrive.car.hyundai.values import CAMERA_SCC_CAR, CANFD_CAR, CAN_GEARS, CAR, CHECKSUM, DATE_FW_ECUS, \ +# EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ +# PLATFORM_CODE_ECUS, get_platform_codes from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR +Ecu = car.CarParams.Ecu +ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} + +# # Some platforms have date codes in a different format we don't yet parse (or are missing). +# # For now, assert list of expected missing date cars +# NO_DATES_PLATFORMS = { +# # CAN FD +# CAR.KIA_SPORTAGE_5TH_GEN, +# CAR.KIA_SPORTAGE_HYBRID_5TH_GEN, +# CAR.SANTA_CRUZ_1ST_GEN, +# CAR.TUCSON_4TH_GEN, +# CAR.TUCSON_HYBRID_4TH_GEN, +# # CAN +# CAR.ELANTRA, +# CAR.KIA_CEED, +# CAR.KIA_FORTE, +# CAR.KIA_OPTIMA_G4, +# CAR.KIA_OPTIMA_G4_FL, +# CAR.KIA_SORENTO, +# CAR.KONA, +# CAR.KONA_EV, +# CAR.KONA_EV_2022, +# CAR.KONA_HEV, +# CAR.SONATA_LF, +# CAR.VELOSTER, +# } + class TestToyotaInterfaces(unittest.TestCase): def test_angle_car_set(self): self.assertTrue(len(ANGLE_CONTROL_CAR - TSS2_CAR) == 0) +class TestToyotaFingerprint(unittest.TestCase): + # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy + # fingerprint in the absence of full FW matches: + # def test_platform_code_ecus_available(self): + # # TODO: add queries for these non-CAN FD cars to get EPS + # no_eps_platforms = CANFD_CAR | {CAR.KIA_SORENTO, CAR.KIA_OPTIMA_G4, CAR.KIA_OPTIMA_G4_FL, + # CAR.SONATA_LF, CAR.TUCSON, CAR.GENESIS_G90, CAR.GENESIS_G80} + # + # # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms + # for car_model, ecus in FW_VERSIONS.items(): + # with self.subTest(car_model=car_model): + # for platform_code_ecu in PLATFORM_CODE_ECUS: + # if platform_code_ecu in (Ecu.fwdRadar, Ecu.eps) and car_model == CAR.HYUNDAI_GENESIS: + # continue + # if platform_code_ecu == Ecu.eps and car_model in no_eps_platforms: + # continue + # self.assertIn(platform_code_ecu, [e[0] for e in ecus]) + # + # def test_fw_format(self): + # # Asserts: + # # - every supported ECU FW version returns one platform code + # # - every supported ECU FW version has a part number + # # - expected parsing of ECU FW dates + # + # for car_model, ecus in FW_VERSIONS.items(): + # with self.subTest(car_model=car_model): + # for ecu, fws in ecus.items(): + # if ecu[0] not in PLATFORM_CODE_ECUS: + # continue + # + # codes = set() + # for fw in fws: + # result = get_platform_codes([fw]) + # self.assertEqual(1, len(result), f"Unable to parse FW: {fw}") + # codes |= result + # + # if ecu[0] not in DATE_FW_ECUS or car_model in NO_DATES_PLATFORMS: + # self.assertTrue(all({date is None for _, date in codes})) + # else: + # self.assertTrue(all({date is not None for _, date in codes})) + # + # if car_model == CAR.HYUNDAI_GENESIS: + # raise unittest.SkipTest("No part numbers for car model") + # + # # Hyundai places the ECU part number in their FW versions, assert all parsable + # # Some examples of valid formats: b"56310-L0010", b"56310L0010", b"56310/M6300" + # self.assertTrue(all({b"-" in code for code, _ in codes}), + # f"FW does not have part number: {fw}") + # + # def test_platform_codes_spot_check(self): + # # Asserts basic platform code parsing behavior for a few cases + # results = get_platform_codes([b"\xf1\x00DH LKAS 1.1 -150210"]) + # self.assertEqual(results, {(b"DH", b"150210")}) + # + # # Some cameras and all radars do not have dates + # results = get_platform_codes([b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 "]) + # self.assertEqual(results, {(b"AEhe-G2000", None)}) + # + # results = get_platform_codes([b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 "]) + # self.assertEqual(results, {(b"CV1-CV000", None)}) + # + # results = get_platform_codes([ + # b"\xf1\x00DH LKAS 1.1 -150210", + # b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 ", + # b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 ", + # ]) + # self.assertEqual(results, {(b"DH", b"150210"), (b"AEhe-G2000", None), (b"CV1-CV000", None)}) + # + # results = get_platform_codes([ + # b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.07 99211-S8100 220222", + # b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.08 99211-S8100 211103", + # b"\xf1\x00ON MFC AT USA LHD 1.00 1.01 99211-S9100 190405", + # b"\xf1\x00ON MFC AT USA LHD 1.00 1.03 99211-S9100 190720", + # ]) + # self.assertEqual(results, {(b"LX2-S8100", b"220222"), (b"LX2-S8100", b"211103"), + # (b"ON-S9100", b"190405"), (b"ON-S9100", b"190720")}) + # + # def test_fuzzy_excluded_platforms(self): + # # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. + # # This list can be shrunk as we combine platforms and detect features + # excluded_platforms = { + # CAR.GENESIS_G70, # shared platform code, part number, and date + # CAR.GENESIS_G70_2020, + # CAR.TUCSON_4TH_GEN, # shared platform code and part number + # CAR.TUCSON_HYBRID_4TH_GEN, + # } + # excluded_platforms |= CANFD_CAR - EV_CAR # shared platform codes + # excluded_platforms |= NO_DATES_PLATFORMS # date codes are required to match + # + # platforms_with_shared_codes = set() + # for platform, fw_by_addr in FW_VERSIONS.items(): + # car_fw = [] + # for ecu, fw_versions in fw_by_addr.items(): + # ecu_name, addr, sub_addr = ecu + # for fw in fw_versions: + # car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr, + # "subAddress": 0 if sub_addr is None else sub_addr}) + # + # CP = car.CarParams.new_message(carFw=car_fw) + # matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) + # if len(matches) == 1: + # self.assertEqual(list(matches)[0], platform) + # else: + # platforms_with_shared_codes.add(platform) + # + # self.assertEqual(platforms_with_shared_codes, excluded_platforms) + + if __name__ == "__main__": unittest.main() diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 58c53b738f0d4f..062a784f16d184 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -1,7 +1,8 @@ +impor re from collections import defaultdict from dataclasses import dataclass from enum import Enum, IntFlag -from typing import Dict, List, Union +from typing import Dict, List, Optional, Set, Tuple, Union from cereal import car from common.conversions import Conversions as CV @@ -215,9 +216,41 @@ class ToyotaCarInfo(CarInfo): (0x4CB, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.AVALON, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'), ] + +def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: + # Returns unique, platform-specific identification codes for a set of versions + codes = set() # (code-Optional[part], date) + for fw in fw_versions: + code_match = PLATFORM_CODE_FW_PATTERN.search(fw) + part_match = PART_NUMBER_FW_PATTERN.search(fw) + date_match = DATE_FW_PATTERN.search(fw) + if code_match is not None: + code: bytes = code_match.group() + part = part_match.group() if part_match else None + date = date_match.group() if date_match else None + if part is not None: + # part number starts with generic ECU part type, add what is specific to platform + code += b"-" + part[-5:] + + codes.add((code, date)) + return codes + + TOYOTA_VERSION_REQUEST_KWP = b'\x1a\x88\x01' TOYOTA_VERSION_RESPONSE_KWP = b'\x5a\x88\x01' +# # Regex patterns for parsing platform code, FW date, and part number from FW versions +# PLATFORM_CODE_FW_PATTERN = re.compile(b'((?<=' + HYUNDAI_VERSION_REQUEST_LONG[1:] + +# b')[A-Z]{2}[A-Za-z0-9]{0,2})') +# DATE_FW_PATTERN = re.compile(b'(?<=[ -])([0-9]{6}$)') +# PART_NUMBER_FW_PATTERN = re.compile(b'(?<=[0-9][.,][0-9]{2} )([0-9]{5}[-/]?[A-Z][A-Z0-9]{3}[0-9])') + +# List of ECUs expected to have platform codes, camera and radar should exist on all cars +# TODO: use abs, it has the platform code and part number on many platforms +PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine] +# So far we've only seen dates in fwdCamera +DATE_FW_ECUS = [Ecu.fwdCamera] + FW_QUERY_CONFIG = FwQueryConfig( requests=[ Request( From 78f812bf43aae02a42db1aa946eb082c97c19c20 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 19 Jun 2023 23:44:15 -0700 Subject: [PATCH 02/65] found tss2 --- selfdrive/car/toyota/tests/test_toyota.py | 22 +++++++++++++++++++++- selfdrive/car/toyota/values.py | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 61553b34bd75f4..f25a44a89e72fc 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -6,7 +6,7 @@ # from selfdrive.car.hyundai.values import CAMERA_SCC_CAR, CANFD_CAR, CAN_GEARS, CAR, CHECKSUM, DATE_FW_ECUS, \ # EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ # PLATFORM_CODE_ECUS, get_platform_codes -from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR +from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -42,6 +42,26 @@ def test_angle_car_set(self): class TestToyotaFingerprint(unittest.TestCase): + def test_fw_debugging(self): + for car_model, ecus in FW_VERSIONS.items(): + print() + print(car_model) + cam_len_code = False + eng_len_code = False + + for ecu, fws in ecus.items(): + if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): + cam_len_code |= all(f[0] < 4 for f in fws) + if ecu[0] in (Ecu.engine, Ecu.abs): + eng_len_code |= all(1 < f[0] < 4 for f in fws) + print(ecu, eng_len_code) + + if (car_model in TSS2_CAR) != cam_len_code: + print(car_model, car_model in TSS2_CAR, cam_len_code) + + if (car_model in EV_HYBRID_CAR) != eng_len_code: + print('MISMATCH', car_model, car_model in EV_HYBRID_CAR, eng_len_code) + # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: # def test_platform_code_ecus_available(self): diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 062a784f16d184..0b0fd40ed0c3be 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -1,4 +1,4 @@ -impor re +import re from collections import defaultdict from dataclasses import dataclass from enum import Enum, IntFlag From f5ddccab1e3850cfb3ded3d2deec7e8367e07586 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 20 Jun 2023 14:10:21 -0700 Subject: [PATCH 03/65] abs and engine are very rarely shared (avalonh tss2 and camryh tss2 is one example) --- selfdrive/car/toyota/tests/test_toyota.py | 57 ++++++++++++++++++----- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index f25a44a89e72fc..064cfeba46414c 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import unittest +from collections import defaultdict from cereal import car from selfdrive.car.fw_versions import build_fw_dict @@ -42,25 +43,57 @@ def test_angle_car_set(self): class TestToyotaFingerprint(unittest.TestCase): - def test_fw_debugging(self): + # def test_fw_debugging(self): + # for car_model, ecus in FW_VERSIONS.items(): + # print() + # print(car_model) + # cam_len_code = False + # eng_len_code = False + # + # for ecu, fws in ecus.items(): + # if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): + # cam_len_code |= all(f[0] < 4 for f in fws) + # if ecu[0] in (Ecu.engine, Ecu.abs): + # eng_len_code |= all(1 < f[0] < 4 for f in fws) + # print(ecu, eng_len_code) + # + # if (car_model in TSS2_CAR) != cam_len_code: + # print(car_model, car_model in TSS2_CAR, cam_len_code) + # + # if (car_model in EV_HYBRID_CAR) != eng_len_code: + # print('MISMATCH', car_model, car_model in EV_HYBRID_CAR, eng_len_code) + + def test_shared_fw(self): + all_fw = defaultdict(set) + for car_model, ecus in FW_VERSIONS.items(): + print() + print(car_model) + + for ecu, fws in ecus.items(): + for fw in fws: + all_fw[(ecu[1], fw)].add(car_model) + + print(all_fw) + + # shared abs (or whatever is in the continue statement) + shared = defaultdict(set) + for car_model, ecus in FW_VERSIONS.items(): print() print(car_model) - cam_len_code = False - eng_len_code = False for ecu, fws in ecus.items(): - if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): - cam_len_code |= all(f[0] < 4 for f in fws) - if ecu[0] in (Ecu.engine, Ecu.abs): - eng_len_code |= all(1 < f[0] < 4 for f in fws) - print(ecu, eng_len_code) + if ecu[0] != Ecu.abs: + continue + for fw in fws: + if len(all_fw[ecu[1], fw]) > 1: + shared[car_model] |= all_fw[ecu[1], fw] + # print(car_model, all_fw[ecu[1], fw]) + # # all_fw[(ecu[1], fw)].add(car_model) - if (car_model in TSS2_CAR) != cam_len_code: - print(car_model, car_model in TSS2_CAR, cam_len_code) + print(shared) + print(len(shared)) - if (car_model in EV_HYBRID_CAR) != eng_len_code: - print('MISMATCH', car_model, car_model in EV_HYBRID_CAR, eng_len_code) # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: From f51c7ac3a91e9c6daafffee7dc45cb0dfbbdd1ef Mon Sep 17 00:00:00 2001 From: sshane Date: Sat, 24 Jun 2023 10:49:45 -0700 Subject: [PATCH 04/65] some bad regex --- selfdrive/car/toyota/tests/test_toyota.py | 60 +++++++++++------------ selfdrive/car/toyota/values.py | 5 ++ 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 064cfeba46414c..d11d1af355fcec 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import re import unittest from collections import defaultdict @@ -7,35 +8,12 @@ # from selfdrive.car.hyundai.values import CAMERA_SCC_CAR, CANFD_CAR, CAN_GEARS, CAR, CHECKSUM, DATE_FW_ECUS, \ # EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ # PLATFORM_CODE_ECUS, get_platform_codes -from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR +from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ + FW_PATTERN, FW_LEN_CODE, FW_PATTERN_V3 Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} -# # Some platforms have date codes in a different format we don't yet parse (or are missing). -# # For now, assert list of expected missing date cars -# NO_DATES_PLATFORMS = { -# # CAN FD -# CAR.KIA_SPORTAGE_5TH_GEN, -# CAR.KIA_SPORTAGE_HYBRID_5TH_GEN, -# CAR.SANTA_CRUZ_1ST_GEN, -# CAR.TUCSON_4TH_GEN, -# CAR.TUCSON_HYBRID_4TH_GEN, -# # CAN -# CAR.ELANTRA, -# CAR.KIA_CEED, -# CAR.KIA_FORTE, -# CAR.KIA_OPTIMA_G4, -# CAR.KIA_OPTIMA_G4_FL, -# CAR.KIA_SORENTO, -# CAR.KONA, -# CAR.KONA_EV, -# CAR.KONA_EV_2022, -# CAR.KONA_HEV, -# CAR.SONATA_LF, -# CAR.VELOSTER, -# } - class TestToyotaInterfaces(unittest.TestCase): def test_angle_car_set(self): @@ -43,6 +21,22 @@ def test_angle_car_set(self): class TestToyotaFingerprint(unittest.TestCase): + def test_fw_pattern(self): + for car_model, ecus in FW_VERSIONS.items(): + # print() + # print(car_model) + + for ecu, fws in ecus.items(): + for fw in fws: + match = FW_PATTERN.search(fw) + length = FW_LEN_CODE.search(fw) + if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): + assert (length is None) == (car_model not in TSS2_CAR), (car_model, ecu, fw) + if ecu[0] in (Ecu.engine,): + assert length is not None + # assert match is not None, (ecu, fw, match) + # print(fw, match) + # def test_fw_debugging(self): # for car_model, ecus in FW_VERSIONS.items(): # print() @@ -66,21 +60,27 @@ class TestToyotaFingerprint(unittest.TestCase): def test_shared_fw(self): all_fw = defaultdict(set) for car_model, ecus in FW_VERSIONS.items(): - print() - print(car_model) + # print() + # print(car_model) for ecu, fws in ecus.items(): + parts = set() for fw in fws: + length, part = FW_PATTERN_V3.search(fw).groups() + parts.add(part) + all_fw[(ecu[1], fw)].add(car_model) + if len(parts) > 1: + print('uoh' ,car_model, ecu, parts) - print(all_fw) + # print(all_fw) # shared abs (or whatever is in the continue statement) shared = defaultdict(set) for car_model, ecus in FW_VERSIONS.items(): - print() - print(car_model) + # print() + # print(car_model) for ecu, fws in ecus.items(): if ecu[0] != Ecu.abs: diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 0b0fd40ed0c3be..df1de2dd2ed7aa 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,6 +280,11 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by } ) +FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') +FW_PATTERN2 = re.compile(br'(?<=\\x[0-9]{2})[0-9A-Z]{5}|^[0-9A-Z]{5}') +FW_LEN_CODE = re.compile(b'^[\x00-\x0F]') +FW_PATTERN_V3 = re.compile(b'(?P^[\x00-\x0F])?(?P[0-9A-Z]{4})') + FW_VERSIONS = { CAR.AVALON: { (Ecu.abs, 0x7b0, None): [ From ae2ecf6f606e0c2eb865689e072b997946e17525 Mon Sep 17 00:00:00 2001 From: sshane Date: Sun, 25 Jun 2023 04:33:05 -0700 Subject: [PATCH 05/65] some pattern work --- selfdrive/car/toyota/tests/test_toyota.py | 5 ++- selfdrive/car/toyota/values.py | 37 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index d11d1af355fcec..60605bd5e3c5d9 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -9,7 +9,7 @@ # EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ # PLATFORM_CODE_ECUS, get_platform_codes from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ - FW_PATTERN, FW_LEN_CODE, FW_PATTERN_V3 + FW_PATTERN, FW_LEN_CODE, FW_PATTERN_V3, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -28,6 +28,8 @@ def test_fw_pattern(self): for ecu, fws in ecus.items(): for fw in fws: + get_platform_codes([fw]) + continue match = FW_PATTERN.search(fw) length = FW_LEN_CODE.search(fw) if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): @@ -58,6 +60,7 @@ def test_fw_pattern(self): # print('MISMATCH', car_model, car_model in EV_HYBRID_CAR, eng_len_code) def test_shared_fw(self): + return all_fw = defaultdict(set) for car_model, ecus in FW_VERSIONS.items(): # print() diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index df1de2dd2ed7aa..814d69762407f0 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -216,11 +216,48 @@ class ToyotaCarInfo(CarInfo): (0x4CB, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.AVALON, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'), ] +SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{4})(?P[A-Z0-9]{4})') +MED_PATTERN = re.compile(b'TODO') +FW_PATTERN = re.compile(b'(?P[0-9A-Z]{4})[0-9A-Z](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: # Returns unique, platform-specific identification codes for a set of versions codes = set() # (code-Optional[part], date) for fw in fw_versions: + has_length = fw[0] < 0xf + length = 1 + if has_length: + length = fw[0] + fw = fw[1:] + assert length * 16 == len(fw) + + chunks = [fw[16 * i:16 * i + 16] for i in range(length)] + print(fw, chunks) + # only first is considered for now since second is commonly shared (TODO: understand that) + + first_chunk = chunks[0] + # doesn't have a part encoded in version (OBD query?) + short_version = sum(b > 0xf for b in first_chunk) == 8 + + + continue + + sections = [s for s in fw.split(b'\x00') if len(s)] + print('sections', sections, 'fw', fw) + assert len(sections) > 0 + assert len(sections[0]) > 0 + + has_length = sections[0][0] < 0xf + if has_length: + assert len(sections) == sections[0][0] + sections[0] = sections[0][1:] + + print(sections) + continue + + + + code_match = PLATFORM_CODE_FW_PATTERN.search(fw) part_match = PART_NUMBER_FW_PATTERN.search(fw) date_match = DATE_FW_PATTERN.search(fw) From 4e1003421a7563f97f8efd1c2022b6bc297aace2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 31 Aug 2023 22:53:20 -0700 Subject: [PATCH 06/65] some work --- selfdrive/car/toyota/values.py | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index bc94ae54d3fa57..eeabbd9c06fff0 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -239,24 +239,52 @@ class ToyotaCarInfo(CarInfo): MED_PATTERN = re.compile(b'TODO') FW_PATTERN = re.compile(b'(?P[0-9A-Z]{4})[0-9A-Z](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') + def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: # Returns unique, platform-specific identification codes for a set of versions codes = set() # (code-Optional[part], date) for fw in fw_versions: - has_length = fw[0] < 0xf - length = 1 - if has_length: - length = fw[0] + # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) + # and are prefixed with a byte that describes how many chunks of data there are. + # But FW returned from KWP requires querying of each sub-data id and does not have a length prefix. + has_n_chunks = fw[0] < 0xf # max seen is 3 chunks, 16 bytes each + n_chunks = 1 + print(f'{has_n_chunks=}') + if has_n_chunks: + n_chunks = fw[0] fw = fw[1:] - assert length * 16 == len(fw) + assert n_chunks * 16 == len(fw) - chunks = [fw[16 * i:16 * i + 16] for i in range(length)] - print(fw, chunks) + chunks = [fw[16 * i:16 * i + 16] for i in range(n_chunks)] + # chunks = [s for s in fw.split(b'\x00') if len(s)] + chunks = [c.strip(b'\x00 ') for c in chunks] + a = list(map(len, chunks)) + print(fw, chunks, a) + # assert len(set(a)) == 1 # only first is considered for now since second is commonly shared (TODO: understand that) first_chunk = chunks[0] # doesn't have a part encoded in version (OBD query?) - short_version = sum(b > 0xf for b in first_chunk) == 8 + # short_version = sum(b > 0xf for b in first_chunk) == 8 + short_version = len(first_chunk) == 8 + if short_version: + print('short version') + code_match = SHORT_FW_PATTERN.search(first_chunk) + if code_match is not None: + code, version = code_match.groups() + print('platform code, version', code, version) + elif len(first_chunk) == 10: + # not done + pass + elif len(first_chunk) == 12: + print(FW_PATTERN) + code_match = FW_PATTERN.search(first_chunk) + if code_match is not None: + print('got long match!') + print(first_chunk, code_match, code_match.groups()) + + else: + assert False, f'invalid length: {len(first_chunk)}' continue @@ -266,8 +294,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by assert len(sections) > 0 assert len(sections[0]) > 0 - has_length = sections[0][0] < 0xf - if has_length: + has_n_chunks = sections[0][0] < 0xf + if has_n_chunks: assert len(sections) == sections[0][0] sections[0] = sections[0][1:] @@ -375,8 +403,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by ], ) -FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') -FW_PATTERN2 = re.compile(br'(?<=\\x[0-9]{2})[0-9A-Z]{5}|^[0-9A-Z]{5}') +# FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') +# FW_PATTERN2 = re.compile(br'(?<=\\x[0-9]{2})[0-9A-Z]{5}|^[0-9A-Z]{5}') FW_LEN_CODE = re.compile(b'^[\x00-\x0F]') FW_PATTERN_V3 = re.compile(b'(?P^[\x00-\x0F])?(?P[0-9A-Z]{4})') From 902fe707451d9d0ad3a734b327995c5dddb7a882 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 31 Aug 2023 22:53:39 -0700 Subject: [PATCH 07/65] . --- selfdrive/car/toyota/tests/test_toyota.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 7d8a292ae1f662..c636b517423e36 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -58,6 +58,8 @@ def test_fw_pattern(self): for ecu, fws in ecus.items(): for fw in fws: + + print('\ninput', fw) get_platform_codes([fw]) continue match = FW_PATTERN.search(fw) From cb84caab733469d48c72fc4416b9a27492e7aaca Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 22:38:43 -0700 Subject: [PATCH 08/65] some conceptual clean up --- selfdrive/car/toyota/tests/test_toyota.py | 14 ++++- selfdrive/car/toyota/values.py | 73 ++++++++++++++++------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index c636b517423e36..ad2ed5fe321589 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from hypothesis import given, settings, strategies as st import re from cereal import car import unittest @@ -10,7 +11,7 @@ # EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ # PLATFORM_CODE_ECUS, get_platform_codes from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ - FW_PATTERN, FW_LEN_CODE, FW_PATTERN_V3, get_platform_codes + LONG_FW_PATTERN, FW_LEN_CODE, get_platform_codes # FW_PATTERN_V3 from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS Ecu = car.CarParams.Ecu @@ -51,6 +52,13 @@ def test_essential_ecus(self): class TestToyotaFingerprint(unittest.TestCase): + # @settings(max_examples=100) + # @given(data=st.data()) + # def test_platform_codes_fuzzy_fw(self, data): + # fw_strategy = st.lists(st.binary()) + # fws = data.draw(fw_strategy) + # get_platform_codes(fws) + def test_fw_pattern(self): for car_model, ecus in FW_VERSIONS.items(): # print() @@ -60,7 +68,9 @@ def test_fw_pattern(self): for fw in fws: print('\ninput', fw) - get_platform_codes([fw]) + ret = get_platform_codes([fw]) + # self.assertTrue(len(ret)) + print('ret', ret) continue match = FW_PATTERN.search(fw) length = FW_LEN_CODE.search(fw) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index eeabbd9c06fff0..9ab572f4cc2c01 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -236,10 +236,12 @@ class ToyotaCarInfo(CarInfo): ] SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{4})(?P[A-Z0-9]{4})') -MED_PATTERN = re.compile(b'TODO') -FW_PATTERN = re.compile(b'(?P[0-9A-Z]{4})[0-9A-Z](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. max seen is 3 chunks, 16 bytes each +import random def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: # Returns unique, platform-specific identification codes for a set of versions codes = set() # (code-Optional[part], date) @@ -247,44 +249,74 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. # But FW returned from KWP requires querying of each sub-data id and does not have a length prefix. - has_n_chunks = fw[0] < 0xf # max seen is 3 chunks, 16 bytes each - n_chunks = 1 - print(f'{has_n_chunks=}') - if has_n_chunks: - n_chunks = fw[0] + # fw = bytearray(fw) + # fw[0] = random.randint(0, 20) + length_code = 1 + length_code_match = FW_LEN_CODE.search(fw) + # has_n_chunks = fw[0] <= 0x5 # max seen is 3 chunks, 16 bytes each + # assert (length_code is not None) == bool(has_n_chunks), fw + # continue + # n_chunks = 1 + # print(f'{has_n_chunks=}') + if length_code_match is not None: + # n_chunks = length_code.group()[0] # fw[0] + length_code = length_code_match.group()[0] fw = fw[1:] - assert n_chunks * 16 == len(fw) - chunks = [fw[16 * i:16 * i + 16] for i in range(n_chunks)] - # chunks = [s for s in fw.split(b'\x00') if len(s)] + # fw length should be multiple of 16 bytes (per chunk, even if no length code), skip parsing if unexpected length + if length_code * 16 != len(fw): + continue + + chunks = [fw[16 * i:16 * i + 16] for i in range(length_code)] chunks = [c.strip(b'\x00 ') for c in chunks] + # chunks = [s.strip(b'\x00 ') for s in fw.split(b'\x00') if len(s)] + + # Ensure not all empty bytes + # TODO: needed since we use + if not len(chunks): + continue + + # if chunks not in ([b'896634A13000', b''], [b'896634A23000', b'']): + # assert chunks == chunks_new, (chunks, chunks_new) + a = list(map(len, chunks)) print(fw, chunks, a) - # assert len(set(a)) == 1 + # assert len(set(a)) == 1, (a, fw) # only first is considered for now since second is commonly shared (TODO: understand that) first_chunk = chunks[0] # doesn't have a part encoded in version (OBD query?) # short_version = sum(b > 0xf for b in first_chunk) == 8 - short_version = len(first_chunk) == 8 - if short_version: + # short_version = len(first_chunk) == 8 + if len(first_chunk) == 8: + # TODO: some short chunks have the part number in subsequent chunks print('short version') code_match = SHORT_FW_PATTERN.search(first_chunk) if code_match is not None: - code, version = code_match.groups() - print('platform code, version', code, version) + platform, version = code_match.groups() + print('platform code, version', platform, version) + codes.add((platform, version)) elif len(first_chunk) == 10: + code_match = MEDIUM_FW_PATTERN.search(first_chunk) + if code_match is not None: + # TODO: platform is a loose term here + part, platform, version = code_match.groups() + codes.add((part + b'-' + platform, version)) + print('not done', first_chunk) # not done pass elif len(first_chunk) == 12: - print(FW_PATTERN) - code_match = FW_PATTERN.search(first_chunk) + print(LONG_FW_PATTERN) + print('long, searching', first_chunk) + code_match = LONG_FW_PATTERN.search(first_chunk) if code_match is not None: print('got long match!') + part, platform, major_version, sub_version = code_match.groups() print(first_chunk, code_match, code_match.groups()) + codes.add((part + b'-' + platform, major_version + b'-' + sub_version)) - else: - assert False, f'invalid length: {len(first_chunk)}' + # else: + # assert False, f'invalid length: {len(first_chunk)}' continue @@ -405,8 +437,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') # FW_PATTERN2 = re.compile(br'(?<=\\x[0-9]{2})[0-9A-Z]{5}|^[0-9A-Z]{5}') -FW_LEN_CODE = re.compile(b'^[\x00-\x0F]') -FW_PATTERN_V3 = re.compile(b'(?P^[\x00-\x0F])?(?P[0-9A-Z]{4})') +# FW_PATTERN_V3 = re.compile(b'(?P^[\x00-\x0F])?(?P[0-9A-Z]{4})') FW_VERSIONS = { CAR.AVALON: { From afec5d7b601a4200c79fe879c69acb63fe410ee1 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:10:54 -0700 Subject: [PATCH 09/65] fix short fw pattern --- selfdrive/car/toyota/values.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 9ab572f4cc2c01..50f9ca6ca3fd95 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -235,16 +235,14 @@ class ToyotaCarInfo(CarInfo): CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'), ] -SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{4})(?P[A-Z0-9]{4})') +SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') -FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. max seen is 3 chunks, 16 bytes each +FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each -import random def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: - # Returns unique, platform-specific identification codes for a set of versions - codes = set() # (code-Optional[part], date) + codes = set() # TODO: standardize (code-Optional[part], date) for fw in fw_versions: # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. From ee881f3bc352bb25f1ef6d81ec2ed098203f595c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:33:53 -0700 Subject: [PATCH 10/65] hyundai test --- selfdrive/car/toyota/tests/test_toyota.py | 83 ++++++++++++++--------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index ad2ed5fe321589..3f09313589a7e3 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -60,6 +60,7 @@ class TestToyotaFingerprint(unittest.TestCase): # get_platform_codes(fws) def test_fw_pattern(self): + return for car_model, ecus in FW_VERSIONS.items(): # print() # print(car_model) @@ -67,9 +68,9 @@ def test_fw_pattern(self): for ecu, fws in ecus.items(): for fw in fws: - print('\ninput', fw) + print('\ninput', car_model, fw) ret = get_platform_codes([fw]) - # self.assertTrue(len(ret)) + self.assertTrue(len(ret)) print('ret', ret) continue match = FW_PATTERN.search(fw) @@ -216,35 +217,55 @@ def test_shared_fw(self): # self.assertEqual(results, {(b"LX2-S8100", b"220222"), (b"LX2-S8100", b"211103"), # (b"ON-S9100", b"190405"), (b"ON-S9100", b"190720")}) # - # def test_fuzzy_excluded_platforms(self): - # # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. - # # This list can be shrunk as we combine platforms and detect features - # excluded_platforms = { - # CAR.GENESIS_G70, # shared platform code, part number, and date - # CAR.GENESIS_G70_2020, - # CAR.TUCSON_4TH_GEN, # shared platform code and part number - # CAR.TUCSON_HYBRID_4TH_GEN, - # } - # excluded_platforms |= CANFD_CAR - EV_CAR # shared platform codes - # excluded_platforms |= NO_DATES_PLATFORMS # date codes are required to match - # - # platforms_with_shared_codes = set() - # for platform, fw_by_addr in FW_VERSIONS.items(): - # car_fw = [] - # for ecu, fw_versions in fw_by_addr.items(): - # ecu_name, addr, sub_addr = ecu - # for fw in fw_versions: - # car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr, - # "subAddress": 0 if sub_addr is None else sub_addr}) - # - # CP = car.CarParams.new_message(carFw=car_fw) - # matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) - # if len(matches) == 1: - # self.assertEqual(list(matches)[0], platform) - # else: - # platforms_with_shared_codes.add(platform) - # - # self.assertEqual(platforms_with_shared_codes, excluded_platforms) + def test_fuzzy_excluded_platforms(self): + # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. + # This list can be shrunk as we combine platforms and detect features + # excluded_platforms = { + # CAR.GENESIS_G70, # shared platform code, part number, and date + # CAR.GENESIS_G70_2020, + # CAR.TUCSON_4TH_GEN, # shared platform code and part number + # CAR.TUCSON_HYBRID_4TH_GEN, + # } + # excluded_platforms |= CANFD_CAR - EV_CAR # shared platform codes + # excluded_platforms |= NO_DATES_PLATFORMS # date codes are required to match + excluded_platforms = set() + + platforms_with_shared_codes = set() + for platform, fw_by_addr in FW_VERSIONS.items(): + # if platform != 'TOYOTA RAV4 2022': + # continue + car_fw = [] + for ecu, fw_versions in fw_by_addr.items(): + ecu_name, addr, sub_addr = ecu + for fw in fw_versions: + car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr, + "subAddress": 0 if sub_addr is None else sub_addr}) + + CP = car.CarParams.new_message(carFw=car_fw) + matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) + if matches in ( + {CAR.RAV4, CAR.RAV4H}, + {CAR.RAV4_TSS2, CAR.RAV4H_TSS2}, + {CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2_2022}, + {CAR.RAV4_TSS2_2023, CAR.RAV4H_TSS2_2023}, + {CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2}, + {CAR.HIGHLANDER, CAR.HIGHLANDERH}, + {CAR.HIGHLANDER_TSS2, CAR.HIGHLANDERH_TSS2}, + {CAR.CAMRY, CAR.CAMRYH}, + {CAR.CAMRY_TSS2, CAR.CAMRYH_TSS2}, + {CAR.CHR, CAR.CHRH}, + {CAR.CHR_TSS2, CAR.CHRH_TSS2}, + {CAR.LEXUS_ES, CAR.LEXUS_ESH}, + {CAR.LEXUS_ES_TSS2, CAR.LEXUS_ESH_TSS2}, + ): + matches = {platform} + print('matches', matches) + if len(matches) == 1: + self.assertEqual(list(matches)[0], platform) + else: + platforms_with_shared_codes.add(platform) + + self.assertEqual(platforms_with_shared_codes, excluded_platforms, (len(platforms_with_shared_codes), len(FW_VERSIONS))) if __name__ == "__main__": From 617930247a4374dd33d69fe16e1c4698cae47cf5 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:34:15 -0700 Subject: [PATCH 11/65] clean up values.py a bit --- selfdrive/car/toyota/values.py | 139 +++++++++++++++++---------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 50f9ca6ca3fd95..2d607c88c41997 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -247,17 +247,10 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. # But FW returned from KWP requires querying of each sub-data id and does not have a length prefix. - # fw = bytearray(fw) - # fw[0] = random.randint(0, 20) + length_code = 1 length_code_match = FW_LEN_CODE.search(fw) - # has_n_chunks = fw[0] <= 0x5 # max seen is 3 chunks, 16 bytes each - # assert (length_code is not None) == bool(has_n_chunks), fw - # continue - # n_chunks = 1 - # print(f'{has_n_chunks=}') if length_code_match is not None: - # n_chunks = length_code.group()[0] # fw[0] length_code = length_code_match.group()[0] fw = fw[1:] @@ -265,89 +258,98 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by if length_code * 16 != len(fw): continue - chunks = [fw[16 * i:16 * i + 16] for i in range(length_code)] - chunks = [c.strip(b'\x00 ') for c in chunks] - # chunks = [s.strip(b'\x00 ') for s in fw.split(b'\x00') if len(s)] + chunks = [fw[16 * i:16 * i + 16].strip(b'\x00 ') for i in range(length_code)] # Ensure not all empty bytes - # TODO: needed since we use if not len(chunks): continue - # if chunks not in ([b'896634A13000', b''], [b'896634A23000', b'']): - # assert chunks == chunks_new, (chunks, chunks_new) + # a = list(map(len, chunks)) + # print(fw, chunks, a) - a = list(map(len, chunks)) - print(fw, chunks, a) - # assert len(set(a)) == 1, (a, fw) # only first is considered for now since second is commonly shared (TODO: understand that) - first_chunk = chunks[0] - # doesn't have a part encoded in version (OBD query?) - # short_version = sum(b > 0xf for b in first_chunk) == 8 - # short_version = len(first_chunk) == 8 if len(first_chunk) == 8: - # TODO: some short chunks have the part number in subsequent chunks - print('short version') - code_match = SHORT_FW_PATTERN.search(first_chunk) - if code_match is not None: - platform, version = code_match.groups() - print('platform code, version', platform, version) - codes.add((platform, version)) + # TODO: no part number, but some short chunks have it in subsequent chunks + # print('short version', fw) + fw_match = SHORT_FW_PATTERN.search(first_chunk) + if fw_match is not None: + platform, major_version, sub_version = fw_match.groups() + # print('platform code, version', platform, major_version, sub_version) + codes.add((platform, major_version)) + elif len(first_chunk) == 10: - code_match = MEDIUM_FW_PATTERN.search(first_chunk) - if code_match is not None: + fw_match = MEDIUM_FW_PATTERN.search(first_chunk) + if fw_match is not None: # TODO: platform is a loose term here - part, platform, version = code_match.groups() + part, platform, version = fw_match.groups() codes.add((part + b'-' + platform, version)) - print('not done', first_chunk) - # not done - pass + elif len(first_chunk) == 12: - print(LONG_FW_PATTERN) - print('long, searching', first_chunk) - code_match = LONG_FW_PATTERN.search(first_chunk) - if code_match is not None: - print('got long match!') - part, platform, major_version, sub_version = code_match.groups() - print(first_chunk, code_match, code_match.groups()) + # print(LONG_FW_PATTERN) + # print('long, searching', first_chunk) + fw_match = LONG_FW_PATTERN.search(first_chunk) + if fw_match is not None: + # print('got long match!') + part, platform, major_version, sub_version = fw_match.groups() + # print(first_chunk, fw_match, fw_match.groups()) codes.add((part + b'-' + platform, major_version + b'-' + sub_version)) - # else: - # assert False, f'invalid length: {len(first_chunk)}' + return codes - continue +def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: + # Non-electric CAN FD platforms often do not have platform code specifiers needed + # to distinguish between hybrid and ICE. All EVs so far are either exclusively + # electric or specify electric in the platform code. + # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers + fuzzy_platform_blacklist = set() # set(CANFD_CAR - EV_CAR) + candidates = set() - sections = [s for s in fw.split(b'\x00') if len(s)] - print('sections', sections, 'fw', fw) - assert len(sections) > 0 - assert len(sections[0]) > 0 + for candidate, fws in FW_VERSIONS.items(): + print('\n\ncandidate:', candidate) + # Keep track of ECUs which pass all checks (platform codes, within date range) + valid_found_ecus = set() + valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS} + for ecu, expected_versions in fws.items(): + addr = ecu[1:] + # Only check ECUs expected to have platform codes + if ecu[0] not in PLATFORM_CODE_ECUS: + continue - has_n_chunks = sections[0][0] < 0xf - if has_n_chunks: - assert len(sections) == sections[0][0] - sections[0] = sections[0][1:] + # Expected platform codes & dates + codes = get_platform_codes(expected_versions) + expected_platform_codes = {code for code, _ in codes} + # expected_dates = {date for _, date in codes if date is not None} - print(sections) - continue + # Found platform codes & dates + codes = get_platform_codes(live_fw_versions.get(addr, set())) + found_platform_codes = {code for code, _ in codes} + # found_dates = {date for _, date in codes if date is not None} + print(ecu, expected_platform_codes, found_platform_codes) + # Check platform code + part number matches for any found versions + if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): + break + # if ecu[0] in DATE_FW_ECUS: + # # If ECU can have a FW date, require it to exist + # # (this excludes candidates in the database without dates) + # if not len(expected_dates) or not len(found_dates): + # break + # + # # Check any date within range in the database, format is %y%m%d + # if not any(min(expected_dates) <= found_date <= max(expected_dates) for found_date in found_dates): + # break + # + valid_found_ecus.add(addr) - code_match = PLATFORM_CODE_FW_PATTERN.search(fw) - part_match = PART_NUMBER_FW_PATTERN.search(fw) - date_match = DATE_FW_PATTERN.search(fw) - if code_match is not None: - code: bytes = code_match.group() - part = part_match.group() if part_match else None - date = date_match.group() if date_match else None - if part is not None: - # part number starts with generic ECU part type, add what is specific to platform - code += b"-" + part[-5:] + # If all live ECUs pass all checks for candidate, add it as a match + if valid_expected_ecus.issubset(valid_found_ecus): + candidates.add(candidate) - codes.add((code, date)) - return codes + return candidates - fuzzy_platform_blacklist # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. @@ -364,9 +366,9 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # List of ECUs expected to have platform codes, camera and radar should exist on all cars # TODO: use abs, it has the platform code and part number on many platforms -PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine] +PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] # So far we've only seen dates in fwdCamera -DATE_FW_ECUS = [Ecu.fwdCamera] +# DATE_FW_ECUS = [Ecu.fwdCamera] FW_QUERY_CONFIG = FwQueryConfig( # TODO: look at data to whitelist new ECUs effectively @@ -431,6 +433,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by (Ecu.combinationMeter, 0x7c0, None), (Ecu.hvac, 0x7c4, None), ], + match_fw_to_car_fuzzy=match_fw_to_car_fuzzy, ) # FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') From 881d3f3633e665b7cd2c7ae945d73621b87272ea Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:34:23 -0700 Subject: [PATCH 12/65] print platform codes --- selfdrive/car/toyota/tests/print_platform_codes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index 1bc8a4e3660e78..e61f681b6d4e35 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from cereal import car -from selfdrive.car.hyundai.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes +from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -10,8 +10,8 @@ print() print(car_model) for ecu in sorted(ecus, key=lambda x: int(x[0])): - if ecu[0] not in PLATFORM_CODE_ECUS: - continue + # if ecu[0] not in PLATFORM_CODE_ECUS: + # continue platform_codes = get_platform_codes(ecus[ecu]) codes = {code for code, _ in platform_codes} From fc3ba3aadfe39bbe6f50fe498515291dcf739d8c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:34:31 -0700 Subject: [PATCH 13/65] hyundai fuzzy --- selfdrive/car/hyundai/tests/test_hyundai.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/selfdrive/car/hyundai/tests/test_hyundai.py b/selfdrive/car/hyundai/tests/test_hyundai.py index 39a9aaf627d945..2b39d32b9b730b 100755 --- a/selfdrive/car/hyundai/tests/test_hyundai.py +++ b/selfdrive/car/hyundai/tests/test_hyundai.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from hypothesis import given, settings, strategies as st import unittest from cereal import car @@ -66,6 +67,13 @@ def test_blacklisted_parts(self): part = code.split(b"-")[1] self.assertFalse(part.startswith(b'CW'), "Car has bad part number") + @settings(max_examples=100) + @given(data=st.data()) + def test_platform_codes_fuzzy_fw(self, data): + fw_strategy = st.lists(st.binary()) + fws = data.draw(fw_strategy) + get_platform_codes(fws) + # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: def test_platform_code_ecus_available(self): From f0450c3427af525a08fc3eacf3c52bc09e626689 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:39:10 -0700 Subject: [PATCH 14/65] pass test --- selfdrive/car/toyota/tests/test_toyota.py | 54 ++++++++++------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 3f09313589a7e3..72469f52d1fbcb 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -60,7 +60,7 @@ class TestToyotaFingerprint(unittest.TestCase): # get_platform_codes(fws) def test_fw_pattern(self): - return + # return for car_model, ecus in FW_VERSIONS.items(): # print() # print(car_model) @@ -219,21 +219,32 @@ def test_shared_fw(self): # def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. - # This list can be shrunk as we combine platforms and detect features - # excluded_platforms = { - # CAR.GENESIS_G70, # shared platform code, part number, and date - # CAR.GENESIS_G70_2020, - # CAR.TUCSON_4TH_GEN, # shared platform code and part number - # CAR.TUCSON_HYBRID_4TH_GEN, - # } - # excluded_platforms |= CANFD_CAR - EV_CAR # shared platform codes - # excluded_platforms |= NO_DATES_PLATFORMS # date codes are required to match - excluded_platforms = set() + # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU + excluded_platforms = { + CAR.LEXUS_ESH_TSS2, + CAR.RAV4_TSS2_2022, + CAR.LEXUS_ES_TSS2, + CAR.RAV4_TSS2, + CAR.RAV4_TSS2_2023, + CAR.CAMRY, + CAR.HIGHLANDER_TSS2, + CAR.RAV4H_TSS2, + CAR.LEXUS_RX_TSS2, + CAR.CAMRYH_TSS2, + CAR.CHR, + CAR.RAV4H, + CAR.RAV4H_TSS2_2022, + CAR.HIGHLANDERH_TSS2, + CAR.RAV4, + CAR.CHR_TSS2, + CAR.CHRH, + CAR.RAV4H_TSS2_2023, + CAR.CAMRY_TSS2, + CAR.COROLLA_TSS2, + } platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): - # if platform != 'TOYOTA RAV4 2022': - # continue car_fw = [] for ecu, fw_versions in fw_by_addr.items(): ecu_name, addr, sub_addr = ecu @@ -243,23 +254,6 @@ def test_fuzzy_excluded_platforms(self): CP = car.CarParams.new_message(carFw=car_fw) matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) - if matches in ( - {CAR.RAV4, CAR.RAV4H}, - {CAR.RAV4_TSS2, CAR.RAV4H_TSS2}, - {CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2_2022}, - {CAR.RAV4_TSS2_2023, CAR.RAV4H_TSS2_2023}, - {CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2}, - {CAR.HIGHLANDER, CAR.HIGHLANDERH}, - {CAR.HIGHLANDER_TSS2, CAR.HIGHLANDERH_TSS2}, - {CAR.CAMRY, CAR.CAMRYH}, - {CAR.CAMRY_TSS2, CAR.CAMRYH_TSS2}, - {CAR.CHR, CAR.CHRH}, - {CAR.CHR_TSS2, CAR.CHRH_TSS2}, - {CAR.LEXUS_ES, CAR.LEXUS_ESH}, - {CAR.LEXUS_ES_TSS2, CAR.LEXUS_ESH_TSS2}, - ): - matches = {platform} - print('matches', matches) if len(matches) == 1: self.assertEqual(list(matches)[0], platform) else: From 1d70d7fb0478fbb1b39dc57d53cd4fb2dec0f5e9 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 7 Sep 2023 23:40:24 -0700 Subject: [PATCH 15/65] move around constants --- selfdrive/car/toyota/values.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 2d607c88c41997..e9b4869861e8fc 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -235,11 +235,6 @@ class ToyotaCarInfo(CarInfo): CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'), ] -SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') -MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') -LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') -FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each - def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: codes = set() # TODO: standardize (code-Optional[part], date) @@ -352,24 +347,23 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: return candidates - fuzzy_platform_blacklist -# Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. -# Toyota diagnostic software first gets the supported data ids, then queries them one by one. -# For example, sends: 0x1a8800, receives: 0x1a8800010203, queries: 0x1a8801, 0x1a8802, 0x1a8803 -TOYOTA_VERSION_REQUEST_KWP = b'\x1a\x88\x01' -TOYOTA_VERSION_RESPONSE_KWP = b'\x5a\x88\x01' - # # Regex patterns for parsing platform code, FW date, and part number from FW versions -# PLATFORM_CODE_FW_PATTERN = re.compile(b'((?<=' + HYUNDAI_VERSION_REQUEST_LONG[1:] + -# b')[A-Z]{2}[A-Za-z0-9]{0,2})') -# DATE_FW_PATTERN = re.compile(b'(?<=[ -])([0-9]{6}$)') -# PART_NUMBER_FW_PATTERN = re.compile(b'(?<=[0-9][.,][0-9]{2} )([0-9]{5}[-/]?[A-Z][A-Z0-9]{3}[0-9])') +SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') +MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each # List of ECUs expected to have platform codes, camera and radar should exist on all cars -# TODO: use abs, it has the platform code and part number on many platforms PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] # So far we've only seen dates in fwdCamera # DATE_FW_ECUS = [Ecu.fwdCamera] +# Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. +# Toyota diagnostic software first gets the supported data ids, then queries them one by one. +# For example, sends: 0x1a8800, receives: 0x1a8800010203, queries: 0x1a8801, 0x1a8802, 0x1a8803 +TOYOTA_VERSION_REQUEST_KWP = b'\x1a\x88\x01' +TOYOTA_VERSION_RESPONSE_KWP = b'\x5a\x88\x01' + FW_QUERY_CONFIG = FwQueryConfig( # TODO: look at data to whitelist new ECUs effectively requests=[ From e05679b38981d715eff4610a8947db4176e5e411 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 13:59:02 -0700 Subject: [PATCH 16/65] clean up --- selfdrive/car/toyota/tests/test_toyota.py | 77 ++--------------------- selfdrive/car/toyota/values.py | 12 ++-- 2 files changed, 8 insertions(+), 81 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 72469f52d1fbcb..b5db0814983d1a 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -11,7 +11,7 @@ # EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ # PLATFORM_CODE_ECUS, get_platform_codes from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ - LONG_FW_PATTERN, FW_LEN_CODE, get_platform_codes # FW_PATTERN_V3 + LONG_FW_PATTERN, FW_LEN_CODE, get_platform_codes from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS Ecu = car.CarParams.Ecu @@ -60,11 +60,8 @@ class TestToyotaFingerprint(unittest.TestCase): # get_platform_codes(fws) def test_fw_pattern(self): - # return + """Asserts all ECUs can be parsed""" for car_model, ecus in FW_VERSIONS.items(): - # print() - # print(car_model) - for ecu, fws in ecus.items(): for fw in fws: @@ -72,74 +69,6 @@ def test_fw_pattern(self): ret = get_platform_codes([fw]) self.assertTrue(len(ret)) print('ret', ret) - continue - match = FW_PATTERN.search(fw) - length = FW_LEN_CODE.search(fw) - if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): - assert (length is None) == (car_model not in TSS2_CAR), (car_model, ecu, fw) - if ecu[0] in (Ecu.engine,): - assert length is not None - # assert match is not None, (ecu, fw, match) - # print(fw, match) - - # def test_fw_debugging(self): - # for car_model, ecus in FW_VERSIONS.items(): - # print() - # print(car_model) - # cam_len_code = False - # eng_len_code = False - # - # for ecu, fws in ecus.items(): - # if ecu[0] in (Ecu.fwdRadar, Ecu.fwdCamera): - # cam_len_code |= all(f[0] < 4 for f in fws) - # if ecu[0] in (Ecu.engine, Ecu.abs): - # eng_len_code |= all(1 < f[0] < 4 for f in fws) - # print(ecu, eng_len_code) - # - # if (car_model in TSS2_CAR) != cam_len_code: - # print(car_model, car_model in TSS2_CAR, cam_len_code) - # - # if (car_model in EV_HYBRID_CAR) != eng_len_code: - # print('MISMATCH', car_model, car_model in EV_HYBRID_CAR, eng_len_code) - - def test_shared_fw(self): - return - all_fw = defaultdict(set) - for car_model, ecus in FW_VERSIONS.items(): - # print() - # print(car_model) - - for ecu, fws in ecus.items(): - parts = set() - for fw in fws: - length, part = FW_PATTERN_V3.search(fw).groups() - parts.add(part) - - all_fw[(ecu[1], fw)].add(car_model) - if len(parts) > 1: - print('uoh' ,car_model, ecu, parts) - - # print(all_fw) - - # shared abs (or whatever is in the continue statement) - shared = defaultdict(set) - - for car_model, ecus in FW_VERSIONS.items(): - # print() - # print(car_model) - - for ecu, fws in ecus.items(): - if ecu[0] != Ecu.abs: - continue - for fw in fws: - if len(all_fw[ecu[1], fw]) > 1: - shared[car_model] |= all_fw[ecu[1], fw] - # print(car_model, all_fw[ecu[1], fw]) - # # all_fw[(ecu[1], fw)].add(car_model) - - print(shared) - print(len(shared)) - # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: @@ -245,6 +174,7 @@ def test_fuzzy_excluded_platforms(self): platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): + print('platform', platform) car_fw = [] for ecu, fw_versions in fw_by_addr.items(): ecu_name, addr, sub_addr = ecu @@ -254,6 +184,7 @@ def test_fuzzy_excluded_platforms(self): CP = car.CarParams.new_message(carFw=car_fw) matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) + print('matches', matches) if len(matches) == 1: self.assertEqual(list(matches)[0], platform) else: diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index e9b4869861e8fc..ed33fe15bd566a 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -302,7 +302,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: candidates = set() for candidate, fws in FW_VERSIONS.items(): - print('\n\ncandidate:', candidate) + # print('\n\ncandidate:', candidate) # Keep track of ECUs which pass all checks (platform codes, within date range) valid_found_ecus = set() valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS} @@ -322,7 +322,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: found_platform_codes = {code for code, _ in codes} # found_dates = {date for _, date in codes if date is not None} - print(ecu, expected_platform_codes, found_platform_codes) + # print(ecu, expected_platform_codes, found_platform_codes) # Check platform code + part number matches for any found versions if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): @@ -347,16 +347,14 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: return candidates - fuzzy_platform_blacklist -# # Regex patterns for parsing platform code, FW date, and part number from FW versions +# Regex patterns for parsing platform code, FW date, and part number from FW versions SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each -# List of ECUs expected to have platform codes, camera and radar should exist on all cars +# List of ECUs expected to have platform codes PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] -# So far we've only seen dates in fwdCamera -# DATE_FW_ECUS = [Ecu.fwdCamera] # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. # Toyota diagnostic software first gets the supported data ids, then queries them one by one. @@ -431,8 +429,6 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: ) # FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') -# FW_PATTERN2 = re.compile(br'(?<=\\x[0-9]{2})[0-9A-Z]{5}|^[0-9A-Z]{5}') -# FW_PATTERN_V3 = re.compile(b'(?P^[\x00-\x0F])?(?P[0-9A-Z]{4})') FW_VERSIONS = { CAR.AVALON: { From 03c14e50ffb20789d6322a92dc897a3858d99c59 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 14:08:04 -0700 Subject: [PATCH 17/65] clean up hyundai --- selfdrive/car/hyundai/tests/test_hyundai.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/selfdrive/car/hyundai/tests/test_hyundai.py b/selfdrive/car/hyundai/tests/test_hyundai.py index 2b39d32b9b730b..39a9aaf627d945 100755 --- a/selfdrive/car/hyundai/tests/test_hyundai.py +++ b/selfdrive/car/hyundai/tests/test_hyundai.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -from hypothesis import given, settings, strategies as st import unittest from cereal import car @@ -67,13 +66,6 @@ def test_blacklisted_parts(self): part = code.split(b"-")[1] self.assertFalse(part.startswith(b'CW'), "Car has bad part number") - @settings(max_examples=100) - @given(data=st.data()) - def test_platform_codes_fuzzy_fw(self, data): - fw_strategy = st.lists(st.binary()) - fws = data.draw(fw_strategy) - get_platform_codes(fws) - # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: def test_platform_code_ecus_available(self): From 24bb20d85f205e462377c9a450f56d20a0710207 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 14:11:44 -0700 Subject: [PATCH 18/65] fix print --- selfdrive/car/toyota/tests/print_platform_codes.py | 8 ++++---- selfdrive/car/toyota/values.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index e61f681b6d4e35..94badc5cde089b 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from cereal import car -from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, get_platform_codes +from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -10,12 +10,12 @@ print() print(car_model) for ecu in sorted(ecus, key=lambda x: int(x[0])): - # if ecu[0] not in PLATFORM_CODE_ECUS: - # continue + if ecu[0] not in PLATFORM_CODE_ECUS: + continue platform_codes = get_platform_codes(ecus[ecu]) codes = {code for code, _ in platform_codes} dates = {date for _, date in platform_codes if date is not None} print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') print(f' Codes: {codes}') - print(f' Dates: {dates}') + print(f' Versions: {dates}') diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index ed33fe15bd566a..645637a4b9957d 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -354,6 +354,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each # List of ECUs expected to have platform codes +# TODO: use hybrid ECU, splits many similar ICE and hybrid variants PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. From 648623ad6572b8dfc9853b97343d880bf5955492 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 14:27:40 -0700 Subject: [PATCH 19/65] more clean up, fix med pattern --- selfdrive/car/toyota/tests/test_toyota.py | 58 +++++++++++------------ selfdrive/car/toyota/values.py | 11 +++-- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index b5db0814983d1a..2b813b343095fe 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -7,9 +7,6 @@ from cereal import car from selfdrive.car.fw_versions import build_fw_dict -# from selfdrive.car.hyundai.values import CAMERA_SCC_CAR, CANFD_CAR, CAN_GEARS, CAR, CHECKSUM, DATE_FW_ECUS, \ -# EV_CAR, FW_QUERY_CONFIG, FW_VERSIONS, LEGACY_SAFETY_MODE_CAR, \ -# PLATFORM_CODE_ECUS, get_platform_codes from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ LONG_FW_PATTERN, FW_LEN_CODE, get_platform_codes from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS @@ -118,33 +115,34 @@ def test_fw_pattern(self): # self.assertTrue(all({b"-" in code for code, _ in codes}), # f"FW does not have part number: {fw}") # - # def test_platform_codes_spot_check(self): - # # Asserts basic platform code parsing behavior for a few cases - # results = get_platform_codes([b"\xf1\x00DH LKAS 1.1 -150210"]) - # self.assertEqual(results, {(b"DH", b"150210")}) - # - # # Some cameras and all radars do not have dates - # results = get_platform_codes([b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 "]) - # self.assertEqual(results, {(b"AEhe-G2000", None)}) - # - # results = get_platform_codes([b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 "]) - # self.assertEqual(results, {(b"CV1-CV000", None)}) - # - # results = get_platform_codes([ - # b"\xf1\x00DH LKAS 1.1 -150210", - # b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 ", - # b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 ", - # ]) - # self.assertEqual(results, {(b"DH", b"150210"), (b"AEhe-G2000", None), (b"CV1-CV000", None)}) - # - # results = get_platform_codes([ - # b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.07 99211-S8100 220222", - # b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.08 99211-S8100 211103", - # b"\xf1\x00ON MFC AT USA LHD 1.00 1.01 99211-S9100 190405", - # b"\xf1\x00ON MFC AT USA LHD 1.00 1.03 99211-S9100 190720", - # ]) - # self.assertEqual(results, {(b"LX2-S8100", b"220222"), (b"LX2-S8100", b"211103"), - # (b"ON-S9100", b"190405"), (b"ON-S9100", b"190720")}) + def test_platform_codes_spot_check(self): + return + # Asserts basic platform code parsing behavior for a few cases + results = get_platform_codes([b'\x018966306L3100\x00\x00\x00\x00']) + self.assertEqual(results, {(b"89663-06", b"L3-100")}) + + # Some cameras and all radars do not have dates + results = get_platform_codes([b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 "]) + self.assertEqual(results, {(b"AEhe-G2000", None)}) + + results = get_platform_codes([b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 "]) + self.assertEqual(results, {(b"CV1-CV000", None)}) + + results = get_platform_codes([ + b"\xf1\x00DH LKAS 1.1 -150210", + b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 ", + b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 ", + ]) + self.assertEqual(results, {(b"DH", b"150210"), (b"AEhe-G2000", None), (b"CV1-CV000", None)}) + + results = get_platform_codes([ + b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.07 99211-S8100 220222", + b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.08 99211-S8100 211103", + b"\xf1\x00ON MFC AT USA LHD 1.00 1.01 99211-S9100 190405", + b"\xf1\x00ON MFC AT USA LHD 1.00 1.03 99211-S9100 190720", + ]) + self.assertEqual(results, {(b"LX2-S8100", b"220222"), (b"LX2-S8100", b"211103"), + (b"ON-S9100", b"190405"), (b"ON-S9100", b"190720")}) # def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 645637a4b9957d..fa8d21d96d4406 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -271,14 +271,15 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by if fw_match is not None: platform, major_version, sub_version = fw_match.groups() # print('platform code, version', platform, major_version, sub_version) - codes.add((platform, major_version)) + codes.add((platform + b'-' + major_version, sub_version)) elif len(first_chunk) == 10: + print('medium fw', fw) fw_match = MEDIUM_FW_PATTERN.search(first_chunk) if fw_match is not None: - # TODO: platform is a loose term here - part, platform, version = fw_match.groups() - codes.add((part + b'-' + platform, version)) + part, platform, major_version, sub_version = fw_match.groups() + # print(part, platform, major_version, sub_version) + codes.add((part + b'-' + platform + b'-' + major_version, sub_version)) elif len(first_chunk) == 12: # print(LONG_FW_PATTERN) @@ -349,7 +350,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # Regex patterns for parsing platform code, FW date, and part number from FW versions SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') -MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each From d194947d91b15bb2228daf9c4956eb372abb3d65 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 15:09:00 -0700 Subject: [PATCH 20/65] add documentation --- selfdrive/car/toyota/values.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index fa8d21d96d4406..0bd10b326ee88a 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -348,7 +348,15 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: return candidates - fuzzy_platform_blacklist -# Regex patterns for parsing platform code, FW date, and part number from FW versions +# Regex patterns for parsing more general platform-specific identifiers from FW versions. +# - Part number: Toyota part number (usually last character needs to be ignored to find a match). +# - Platform: usually multiple codes per an openpilot platform, however this has the less variability and +# is usually shared across ECUs and model years signifying this describes something about the specific platform. +# - Major version: second least variable part of the FW version. Seen splitting cars by model year such as RAV4 2022/2023 and Prius. +# It is important to note that these aren't always consecutive, for example: +# Prius TSS-P has these major versions over 16 FW: 2, 3, 4, 6, 8 while Prius TSS2 has: 5 +# - Sub version: exclusive to major version, but shared with other cars. Should only be used for further filtering, +# more exploration is needed. SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') From 780a66fc5f4aab8164644b30b0a8129f8247ccf7 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 8 Sep 2023 15:14:41 -0700 Subject: [PATCH 21/65] use major version --- selfdrive/car/toyota/tests/test_toyota.py | 41 +++++++++++++---------- selfdrive/car/toyota/values.py | 7 ++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 2b813b343095fe..f51e2e9ae52a70 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -148,30 +148,35 @@ def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU excluded_platforms = { - CAR.LEXUS_ESH_TSS2, - CAR.RAV4_TSS2_2022, + # # CAR.LEXUS_ESH_TSS2, + # # CAR.RAV4_TSS2_2022, CAR.LEXUS_ES_TSS2, - CAR.RAV4_TSS2, - CAR.RAV4_TSS2_2023, - CAR.CAMRY, - CAR.HIGHLANDER_TSS2, - CAR.RAV4H_TSS2, + # # CAR.RAV4_TSS2, + # # CAR.RAV4_TSS2_2023, + # CAR.RAV4_TSS2, + # # CAR.CAMRY, + # CAR.HIGHLANDER_TSS2, + # CAR.RAV4H_TSS2, CAR.LEXUS_RX_TSS2, - CAR.CAMRYH_TSS2, - CAR.CHR, - CAR.RAV4H, - CAR.RAV4H_TSS2_2022, - CAR.HIGHLANDERH_TSS2, - CAR.RAV4, - CAR.CHR_TSS2, - CAR.CHRH, - CAR.RAV4H_TSS2_2023, - CAR.CAMRY_TSS2, - CAR.COROLLA_TSS2, + # # CAR.CAMRYH_TSS2, + # CAR.CHR, + # # CAR.RAV4H, + # CAR.RAV4H_TSS2_2022, + # # CAR.HIGHLANDERH_TSS2, + # # CAR.RAV4, + # # CAR.CHR_TSS2, + # CAR.CHRH, + # CAR.RAV4H_TSS2_2023, + # CAR.CAMRY_TSS2, + # CAR.COROLLA_TSS2, } platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): + # if platform != CAR.RAV4_TSS2: + # continue + # if 'RAV4 HYBRID' not in platform: + # continue print('platform', platform) car_fw = [] for ecu, fw_versions in fw_by_addr.items(): diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 0bd10b326ee88a..ecd125cda8c708 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -259,9 +259,6 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by if not len(chunks): continue - # a = list(map(len, chunks)) - # print(fw, chunks, a) - # only first is considered for now since second is commonly shared (TODO: understand that) first_chunk = chunks[0] if len(first_chunk) == 8: @@ -274,7 +271,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by codes.add((platform + b'-' + major_version, sub_version)) elif len(first_chunk) == 10: - print('medium fw', fw) + # print('medium fw', fw) fw_match = MEDIUM_FW_PATTERN.search(first_chunk) if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() @@ -289,7 +286,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # print('got long match!') part, platform, major_version, sub_version = fw_match.groups() # print(first_chunk, fw_match, fw_match.groups()) - codes.add((part + b'-' + platform, major_version + b'-' + sub_version)) + codes.add((part + b'-' + platform + b'-' + major_version, sub_version)) return codes From 36e519debd865e2e95249478967be9b79025bcd2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 10 Sep 2023 20:58:46 -0700 Subject: [PATCH 22/65] some clean up from merge --- selfdrive/car/toyota/tests/test_toyota.py | 33 ++++++----------------- selfdrive/car/toyota/values.py | 16 ++++------- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index d494a7129bffa6..73f3ff8ff49cc7 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -16,8 +16,6 @@ Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} -ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} - class TestToyotaInterfaces(unittest.TestCase): def test_car_sets(self): @@ -52,14 +50,14 @@ def test_essential_ecus(self): class TestToyotaFingerprint(unittest.TestCase): - # @settings(max_examples=100) - # @given(data=st.data()) - # def test_platform_codes_fuzzy_fw(self, data): - # fw_strategy = st.lists(st.binary()) - # fws = data.draw(fw_strategy) - # get_platform_codes(fws) - - def test_fw_pattern(self): + @settings(max_examples=100) + @given(data=st.data()) + def test_platform_codes_fuzzy_fw(self, data): + fw_strategy = st.lists(st.binary()) + fws = data.draw(fw_strategy) + get_platform_codes(fws) + + def test_fw_pattern_new(self): """Asserts all ECUs can be parsed""" for car_model, ecus in FW_VERSIONS.items(): for ecu, fws in ecus.items(): @@ -198,21 +196,6 @@ def test_fuzzy_excluded_platforms(self): self.assertEqual(platforms_with_shared_codes, excluded_platforms, (len(platforms_with_shared_codes), len(FW_VERSIONS))) - @settings(max_examples=100) - @given(data=st.data()) - def test_platform_codes_fuzzy_fw(self, data): - fw_strategy = st.lists(st.binary()) - fws = data.draw(fw_strategy) - get_platform_codes(fws) - - def test_fw_pattern(self): - """Asserts all ECUs can be parsed""" - for ecus in FW_VERSIONS.values(): - for fws in ecus.values(): - for fw in fws: - ret = get_platform_codes([fw]) - self.assertTrue(len(ret)) - if __name__ == "__main__": unittest.main() diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 09d3c69c07e0ad..7a6dcbb6c1f999 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -250,14 +250,10 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by fw = fw[1:] # fw length should be multiple of 16 bytes (per chunk, even if no length code), skip parsing if unexpected length - if length_code * 16 != len(fw): + if length_code * FW_CHUNK_LEN != len(fw): continue - chunks = [fw[16 * i:16 * i + 16].strip(b'\x00 ') for i in range(length_code)] - - # Ensure not all empty bytes - if not len(chunks): - continue + chunks = [fw[FW_CHUNK_LEN * i:FW_CHUNK_LEN * i + FW_CHUNK_LEN].strip(b'\x00 ') for i in range(length_code)] # only first is considered for now since second is commonly shared (TODO: understand that) first_chunk = chunks[0] @@ -267,8 +263,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by fw_match = SHORT_FW_PATTERN.search(first_chunk) if fw_match is not None: platform, major_version, sub_version = fw_match.groups() + codes.add((b'-'.join((platform, major_version)), sub_version)) # print('platform code, version', platform, major_version, sub_version) - codes.add((platform + b'-' + major_version, sub_version)) elif len(first_chunk) == 10: # print('medium fw', fw) @@ -276,7 +272,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() # print(part, platform, major_version, sub_version) - codes.add((part + b'-' + platform + b'-' + major_version, sub_version)) + codes.add((b'-'.join((part, platform, major_version)), sub_version)) elif len(first_chunk) == 12: # print(LONG_FW_PATTERN) @@ -286,7 +282,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[by # print('got long match!') part, platform, major_version, sub_version = fw_match.groups() # print(first_chunk, fw_match, fw_match.groups()) - codes.add((part + b'-' + platform + b'-' + major_version, sub_version)) + codes.add((b'-'.join((part, platform, major_version)), sub_version)) return codes @@ -437,8 +433,6 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: match_fw_to_car_fuzzy=match_fw_to_car_fuzzy, ) -# FW_PATTERN = re.compile(b'[0-9]{4}[0-9A-Z][0-9A-Z]') - FW_VERSIONS = { CAR.AVALON: { (Ecu.abs, 0x7b0, None): [ From 94e7272746a5bd1d9f4902d81d9d8d4755e29432 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 10 Sep 2023 20:59:29 -0700 Subject: [PATCH 23/65] some clean up from merge 2 --- selfdrive/car/toyota/values.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 7a6dcbb6c1f999..91f528670bb54a 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -236,8 +236,8 @@ class ToyotaCarInfo(CarInfo): ] -def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, Optional[bytes]]]: - codes = set() # TODO: standardize (code-Optional[part], date) +def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: + codes = set() # (Optional[part]-platform-major_version, minor_version) for fw in fw_versions: # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. From 02f3e2ff58ba1f15a235b568c0555faaed9cc139 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 10 Sep 2023 21:11:20 -0700 Subject: [PATCH 24/65] spot check --- selfdrive/car/toyota/tests/test_toyota.py | 34 ++++++++--------------- selfdrive/car/toyota/values.py | 3 +- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 73f3ff8ff49cc7..36f72eaca23b01 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -115,36 +115,26 @@ def test_fw_pattern_new(self): # # Some examples of valid formats: b"56310-L0010", b"56310L0010", b"56310/M6300" # self.assertTrue(all({b"-" in code for code, _ in codes}), # f"FW does not have part number: {fw}") - # + def test_platform_codes_spot_check(self): - return # Asserts basic platform code parsing behavior for a few cases - results = get_platform_codes([b'\x018966306L3100\x00\x00\x00\x00']) - self.assertEqual(results, {(b"89663-06", b"L3-100")}) + results = get_platform_codes([b"F152607140\x00\x00\x00\x00\x00\x00"]) + self.assertEqual(results, {(b"F1526-07-1", b"40")}) - # Some cameras and all radars do not have dates - results = get_platform_codes([b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 "]) - self.assertEqual(results, {(b"AEhe-G2000", None)}) + results = get_platform_codes([b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00"]) + self.assertEqual(results, {(b"8646F-41-04", b"100")}) - results = get_platform_codes([b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 "]) - self.assertEqual(results, {(b"CV1-CV000", None)}) + # Short version has no part number + results = get_platform_codes([b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00"]) + self.assertEqual(results, {(b"35-87", b"9000")}) results = get_platform_codes([ - b"\xf1\x00DH LKAS 1.1 -150210", - b"\xf1\x00AEhe SCC H-CUP 1.01 1.01 96400-G2000 ", - b"\xf1\x00CV1_ RDR ----- 1.00 1.01 99110-CV000 ", + b"F152607140\x00\x00\x00\x00\x00\x00", + b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", + b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00", ]) - self.assertEqual(results, {(b"DH", b"150210"), (b"AEhe-G2000", None), (b"CV1-CV000", None)}) + self.assertEqual(results, {(b"F1526-07-1", b"40"), (b"8646F-41-04", b"100"), (b"35-87", b"9000")}) - results = get_platform_codes([ - b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.07 99211-S8100 220222", - b"\xf1\x00LX2 MFC AT USA LHD 1.00 1.08 99211-S8100 211103", - b"\xf1\x00ON MFC AT USA LHD 1.00 1.01 99211-S9100 190405", - b"\xf1\x00ON MFC AT USA LHD 1.00 1.03 99211-S9100 190720", - ]) - self.assertEqual(results, {(b"LX2-S8100", b"220222"), (b"LX2-S8100", b"211103"), - (b"ON-S9100", b"190405"), (b"ON-S9100", b"190720")}) - # def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 91f528670bb54a..4d1a572496a43d 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -2,7 +2,7 @@ from collections import defaultdict from dataclasses import dataclass, field from enum import Enum, IntFlag -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Set, Tuple, Union from cereal import car from openpilot.common.conversions import Conversions as CV @@ -357,7 +357,6 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: FW_CHUNK_LEN = 16 # List of ECUs expected to have platform codes -# TODO: use hybrid ECU, splits many similar ICE and hybrid variants PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] From 0b9f67229bbfc92a2cc104e15840094f473ea252 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 10 Sep 2023 21:12:27 -0700 Subject: [PATCH 25/65] clean up imports --- selfdrive/car/toyota/tests/test_toyota.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 36f72eaca23b01..2a7f6fd92954c6 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,15 +1,9 @@ #!/usr/bin/env python3 from hypothesis import given, settings, strategies as st -import re -from cereal import car import unittest -from collections import defaultdict from cereal import car from selfdrive.car.fw_versions import build_fw_dict -from selfdrive.car.toyota.values import TSS2_CAR, ANGLE_CONTROL_CAR, FW_VERSIONS, FW_QUERY_CONFIG, EV_HYBRID_CAR, \ - LONG_FW_PATTERN, FW_LEN_CODE, get_platform_codes -from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ get_platform_codes From 4900e220b3feef35c3b5b0eb20d69856ea444d0b Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 10 Sep 2023 21:24:44 -0700 Subject: [PATCH 26/65] missing FW_QUERY_CONFIG --- selfdrive/car/toyota/tests/test_toyota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 2a7f6fd92954c6..5065c4015bbb3c 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -5,7 +5,7 @@ from cereal import car from selfdrive.car.fw_versions import build_fw_dict from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ - get_platform_codes + FW_QUERY_CONFIG, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} From fd884c82752429a5733d69db738367277af84d6a Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 22:38:31 -0700 Subject: [PATCH 27/65] short version: always prefixed with 3, get real platform code --- selfdrive/car/toyota/values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index ba516f7cc892ec..8e5cd4f0107a4b 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -350,7 +350,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # Prius TSS-P has these major versions over 16 FW: 2, 3, 4, 6, 8 while Prius TSS2 has: 5 # - Sub version: exclusive to major version, but shared with other cars. Should only be used for further filtering, # more exploration is needed. -SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') +SHORT_FW_PATTERN = re.compile(b'[A-Z0-9](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each From 0a1c00fbb3dc20a94ee4d8d96bf83cc232e7168b Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:02:53 -0700 Subject: [PATCH 28/65] limit to max chunks seen (3) --- selfdrive/car/toyota/values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 8e5cd4f0107a4b..5b1fd0d2dd1b2f 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -353,7 +353,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: SHORT_FW_PATTERN = re.compile(b'[A-Z0-9](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') -FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each +FW_LEN_CODE = re.compile(b'^[\x01-\x03]') # highest seen is 3 chunks, 16 bytes each FW_CHUNK_LEN = 16 # List of ECUs expected to have platform codes From a462a914fd880f865a0f7c9abe1c143ce1f1462e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:10:58 -0700 Subject: [PATCH 29/65] rm engine --- selfdrive/car/toyota/values.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 5b1fd0d2dd1b2f..e3f14cfdbe7b4e 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -357,7 +357,8 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: FW_CHUNK_LEN = 16 # List of ECUs expected to have platform codes -PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] +# Engine is not well understood at this time (lots of variability in major version) +PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. From 0dd965531314253294bac54a4c14be97898f1bdf Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:11:42 -0700 Subject: [PATCH 30/65] fix that --- selfdrive/car/toyota/tests/test_toyota.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 5065c4015bbb3c..2c325e347aca30 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -120,14 +120,14 @@ def test_platform_codes_spot_check(self): # Short version has no part number results = get_platform_codes([b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00"]) - self.assertEqual(results, {(b"35-87", b"9000")}) + self.assertEqual(results, {(b"58-79", b"000")}) results = get_platform_codes([ b"F152607140\x00\x00\x00\x00\x00\x00", b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00", ]) - self.assertEqual(results, {(b"F1526-07-1", b"40"), (b"8646F-41-04", b"100"), (b"35-87", b"9000")}) + self.assertEqual(results, {(b"F1526-07-1", b"40"), (b"8646F-41-04", b"100"), (b"58-79", b"000")}) def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. From 90ed88209bff87820fee54d2555cf9e8d06e1bde Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:52:00 -0700 Subject: [PATCH 31/65] get_platform_codes returns dict --- selfdrive/car/toyota/values.py | 47 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index e3f14cfdbe7b4e..eb39406903cc50 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -236,8 +236,9 @@ class ToyotaCarInfo(CarInfo): ] -def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: - codes = set() # (Optional[part]-platform-major_version, minor_version) +def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: + # Returns minor versions in a dict so comparisons can be made within part-platform-version combos + codes = defaultdict(set) # Optional[part]-platform-major_version: set of minor_version for fw in fw_versions: # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. @@ -263,7 +264,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: fw_match = SHORT_FW_PATTERN.search(first_chunk) if fw_match is not None: platform, major_version, sub_version = fw_match.groups() - codes.add((b'-'.join((platform, major_version)), sub_version)) + codes[b'-'.join((platform, major_version))].add(sub_version) # print('platform code, version', platform, major_version, sub_version) elif len(first_chunk) == 10: @@ -272,19 +273,19 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() # print(part, platform, major_version, sub_version) - codes.add((b'-'.join((part, platform, major_version)), sub_version)) + codes[b'-'.join((part, platform, major_version))].add(sub_version) elif len(first_chunk) == 12: # print(LONG_FW_PATTERN) - # print('long, searching', first_chunk) + # print('long version', fw) fw_match = LONG_FW_PATTERN.search(first_chunk) if fw_match is not None: # print('got long match!') part, platform, major_version, sub_version = fw_match.groups() # print(first_chunk, fw_match, fw_match.groups()) - codes.add((b'-'.join((part, platform, major_version)), sub_version)) + codes[b'-'.join((part, platform, major_version))].add(sub_version) - return codes + return dict(codes) def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: @@ -307,32 +308,24 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: continue # Expected platform codes & dates - codes = get_platform_codes(expected_versions) - expected_platform_codes = {code for code, _ in codes} - # expected_dates = {date for _, date in codes if date is not None} + expected_platform_codes = get_platform_codes(expected_versions) # Found platform codes & dates - codes = get_platform_codes(live_fw_versions.get(addr, set())) - found_platform_codes = {code for code, _ in codes} - # found_dates = {date for _, date in codes if date is not None} + found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set())) # print(ecu, expected_platform_codes, found_platform_codes) - # Check platform code + part number matches for any found versions - if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): - break + # Check part number + platform code + major version matches for any found versions + # Then check that sub-version for the above is within range (splits model years) + for found_platform_code, found_sub_versions in found_platform_codes.items(): + if found_platform_code in expected_platform_codes: + expected_sub_versions = expected_platform_codes[found_platform_code] - # if ecu[0] in DATE_FW_ECUS: - # # If ECU can have a FW date, require it to exist - # # (this excludes candidates in the database without dates) - # if not len(expected_dates) or not len(found_dates): - # break - # - # # Check any date within range in the database, format is %y%m%d - # if not any(min(expected_dates) <= found_date <= max(expected_dates) for found_date in found_dates): - # break - # - valid_found_ecus.add(addr) + # Check any sub-version within range in the database for this part-platform-version combo + if any(min(expected_sub_versions) <= found_sub_version <= max(expected_sub_versions) for + found_sub_version in found_sub_versions): + valid_found_ecus.add(addr) + break # If all live ECUs pass all checks for candidate, add it as a match if valid_expected_ecus.issubset(valid_found_ecus): From c1e60c96c144a31c560ead1c36620eaeb426629e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:53:03 -0700 Subject: [PATCH 32/65] tests --- selfdrive/car/toyota/tests/test_toyota.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 2c325e347aca30..2033b358988a78 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -5,7 +5,7 @@ from cereal import car from selfdrive.car.fw_versions import build_fw_dict from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ - FW_QUERY_CONFIG, get_platform_codes + FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -56,6 +56,8 @@ def test_fw_pattern_new(self): for car_model, ecus in FW_VERSIONS.items(): for ecu, fws in ecus.items(): for fw in fws: + if ecu[0] not in PLATFORM_CODE_ECUS: + continue print('\ninput', car_model, fw) ret = get_platform_codes([fw]) @@ -144,7 +146,7 @@ def test_fuzzy_excluded_platforms(self): # CAR.RAV4H_TSS2, CAR.LEXUS_RX_TSS2, # # CAR.CAMRYH_TSS2, - # CAR.CHR, + CAR.CHR, # # CAR.RAV4H, # CAR.RAV4H_TSS2_2022, # # CAR.HIGHLANDERH_TSS2, @@ -153,7 +155,7 @@ def test_fuzzy_excluded_platforms(self): # CAR.CHRH, # CAR.RAV4H_TSS2_2023, # CAR.CAMRY_TSS2, - # CAR.COROLLA_TSS2, + CAR.COROLLA_TSS2, } platforms_with_shared_codes = set() From 391eb22842027b047dfb06deee2da973eb626490 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 17 Sep 2023 23:54:24 -0700 Subject: [PATCH 33/65] comments --- selfdrive/car/toyota/values.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index eb39406903cc50..90d24b501684ec 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -298,7 +298,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: for candidate, fws in FW_VERSIONS.items(): # print('\n\ncandidate:', candidate) - # Keep track of ECUs which pass all checks (platform codes, within date range) + # Keep track of ECUs which pass all checks (platform codes, within sub-version range) valid_found_ecus = set() valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS} for ecu, expected_versions in fws.items(): @@ -307,10 +307,10 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if ecu[0] not in PLATFORM_CODE_ECUS: continue - # Expected platform codes & dates + # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) - # Found platform codes & dates + # Found platform codes & versions found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set())) # print(ecu, expected_platform_codes, found_platform_codes) From e700568bfd1e9c3e25a6af6a9634bf3c350566e7 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:03:53 -0700 Subject: [PATCH 34/65] fix test --- selfdrive/car/toyota/tests/test_toyota.py | 25 ++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 2033b358988a78..57a7945fb1b6a2 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -114,22 +114,33 @@ def test_fw_pattern_new(self): def test_platform_codes_spot_check(self): # Asserts basic platform code parsing behavior for a few cases - results = get_platform_codes([b"F152607140\x00\x00\x00\x00\x00\x00"]) - self.assertEqual(results, {(b"F1526-07-1", b"40")}) + results = get_platform_codes([ + b"F152607140\x00\x00\x00\x00\x00\x00", + b"F152607171\x00\x00\x00\x00\x00\x00", + b"F152607110\x00\x00\x00\x00\x00\x00", + b"F152607180\x00\x00\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"F1526-07-1": {b"10", b"40", b"71", b"80"}}) - results = get_platform_codes([b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00"]) - self.assertEqual(results, {(b"8646F-41-04", b"100")}) + results = get_platform_codes([ + b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", + b"\x028646F4104100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"8646F-41-04": {b"100"}}) # Short version has no part number - results = get_platform_codes([b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00"]) - self.assertEqual(results, {(b"58-79", b"000")}) + results = get_platform_codes([ + b"\x0235870000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00", + b"\x0235883000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"58-70": {b"000"}, b"58-83": {b"000"}}) results = get_platform_codes([ b"F152607140\x00\x00\x00\x00\x00\x00", b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00", ]) - self.assertEqual(results, {(b"F1526-07-1", b"40"), (b"8646F-41-04", b"100"), (b"58-79", b"000")}) + self.assertEqual(results, {b"F1526-07-1": {b"40"}, b"8646F-41-04": {b"100"}, b"58-79": {b"000"}}) def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. From 7fb9dc430af0e23b20e560252de0d3a17970b46c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:22:28 -0700 Subject: [PATCH 35/65] enable a test --- selfdrive/car/toyota/tests/test_toyota.py | 33 +++++++++++------------ 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 57a7945fb1b6a2..d282c54ae20a6d 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -44,6 +44,8 @@ def test_essential_ecus(self): class TestToyotaFingerprint(unittest.TestCase): + # Tests for part numbers, platform codes, and sub-versions which Toyota will use to fuzzy + # fingerprint in the absence of full FW matches: @settings(max_examples=100) @given(data=st.data()) def test_platform_codes_fuzzy_fw(self, data): @@ -64,23 +66,20 @@ def test_fw_pattern_new(self): self.assertTrue(len(ret)) print('ret', ret) - # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy - # fingerprint in the absence of full FW matches: - # def test_platform_code_ecus_available(self): - # # TODO: add queries for these non-CAN FD cars to get EPS - # no_eps_platforms = CANFD_CAR | {CAR.KIA_SORENTO, CAR.KIA_OPTIMA_G4, CAR.KIA_OPTIMA_G4_FL, - # CAR.SONATA_LF, CAR.TUCSON, CAR.GENESIS_G90, CAR.GENESIS_G80} - # - # # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms - # for car_model, ecus in FW_VERSIONS.items(): - # with self.subTest(car_model=car_model): - # for platform_code_ecu in PLATFORM_CODE_ECUS: - # if platform_code_ecu in (Ecu.fwdRadar, Ecu.eps) and car_model == CAR.HYUNDAI_GENESIS: - # continue - # if platform_code_ecu == Ecu.eps and car_model in no_eps_platforms: - # continue - # self.assertIn(platform_code_ecu, [e[0] for e in ecus]) - # + def test_platform_code_ecus_available(self): + # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms + for car_model, ecus in FW_VERSIONS.items(): + with self.subTest(car_model=car_model): + for platform_code_ecu in PLATFORM_CODE_ECUS: + if platform_code_ecu == Ecu.eps and car_model in (CAR.PRIUS_V, CAR.LEXUS_CTH,): + continue + if platform_code_ecu == Ecu.abs and car_model in (CAR.ALPHARD_TSS2,): + continue + # TODO: add DSU FW versions for Highlander Hybrid + if platform_code_ecu == Ecu.dsu and car_model in TSS2_CAR | {CAR.HIGHLANDERH}: + continue + self.assertIn(platform_code_ecu, [e[0] for e in ecus]) + # def test_fw_format(self): # # Asserts: # # - every supported ECU FW version returns one platform code From 36c43b85fe792ec2478976b7f65388d3a2f137c2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:29:05 -0700 Subject: [PATCH 36/65] fix script --- selfdrive/car/toyota/tests/print_platform_codes.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index 94badc5cde089b..dc2f6513129291 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -14,8 +14,5 @@ continue platform_codes = get_platform_codes(ecus[ecu]) - codes = {code for code, _ in platform_codes} - dates = {date for _, date in platform_codes if date is not None} print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') - print(f' Codes: {codes}') - print(f' Versions: {dates}') + print(f' Codes: {platform_codes}') From 99da7351046fc48a5815ff8b00669b8d8d69eaa1 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:43:35 -0700 Subject: [PATCH 37/65] print ecu parts --- selfdrive/car/toyota/tests/print_platform_codes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index dc2f6513129291..25b921819df857 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from collections import defaultdict from cereal import car from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes @@ -6,6 +7,8 @@ ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} if __name__ == "__main__": + parts_for_ecu = defaultdict(set) + for car_model, ecus in FW_VERSIONS.items(): print() print(car_model) @@ -14,5 +17,10 @@ continue platform_codes = get_platform_codes(ecus[ecu]) + parts_for_ecu[ecu] |= {code.split(b'-')[0] for code in platform_codes} print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') print(f' Codes: {platform_codes}') + + print('\nECU parts:') + for ecu, parts in parts_for_ecu.items(): + print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}): {parts}') From 16aef10003b598d69447a5c9f5dfeef39cca93a6 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:50:43 -0700 Subject: [PATCH 38/65] enable old test --- selfdrive/car/toyota/tests/test_toyota.py | 68 +++++++++-------------- 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index d282c54ae20a6d..79e23ab8ef46c4 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -53,19 +53,6 @@ def test_platform_codes_fuzzy_fw(self, data): fws = data.draw(fw_strategy) get_platform_codes(fws) - def test_fw_pattern_new(self): - """Asserts all ECUs can be parsed""" - for car_model, ecus in FW_VERSIONS.items(): - for ecu, fws in ecus.items(): - for fw in fws: - if ecu[0] not in PLATFORM_CODE_ECUS: - continue - - print('\ninput', car_model, fw) - ret = get_platform_codes([fw]) - self.assertTrue(len(ret)) - print('ret', ret) - def test_platform_code_ecus_available(self): # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms for car_model, ecus in FW_VERSIONS.items(): @@ -80,36 +67,31 @@ def test_platform_code_ecus_available(self): continue self.assertIn(platform_code_ecu, [e[0] for e in ecus]) - # def test_fw_format(self): - # # Asserts: - # # - every supported ECU FW version returns one platform code - # # - every supported ECU FW version has a part number - # # - expected parsing of ECU FW dates - # - # for car_model, ecus in FW_VERSIONS.items(): - # with self.subTest(car_model=car_model): - # for ecu, fws in ecus.items(): - # if ecu[0] not in PLATFORM_CODE_ECUS: - # continue - # - # codes = set() - # for fw in fws: - # result = get_platform_codes([fw]) - # self.assertEqual(1, len(result), f"Unable to parse FW: {fw}") - # codes |= result - # - # if ecu[0] not in DATE_FW_ECUS or car_model in NO_DATES_PLATFORMS: - # self.assertTrue(all({date is None for _, date in codes})) - # else: - # self.assertTrue(all({date is not None for _, date in codes})) - # - # if car_model == CAR.HYUNDAI_GENESIS: - # raise unittest.SkipTest("No part numbers for car model") - # - # # Hyundai places the ECU part number in their FW versions, assert all parsable - # # Some examples of valid formats: b"56310-L0010", b"56310L0010", b"56310/M6300" - # self.assertTrue(all({b"-" in code for code, _ in codes}), - # f"FW does not have part number: {fw}") + def test_fw_format(self): + # Asserts: + # - every supported ECU FW version returns one platform code + # - every supported ECU FW version has a part number + # - expected parsing of ECU sub-versions + + for car_model, ecus in FW_VERSIONS.items(): + with self.subTest(car_model=car_model): + for ecu, fws in ecus.items(): + if ecu[0] not in PLATFORM_CODE_ECUS: + continue + + codes = dict() + for fw in fws: + result = get_platform_codes([fw]) + # Check only one platform code and sub-version + self.assertEqual(1, len(result), f"Unable to parse FW: {fw}") + self.assertEqual(1, len(list(result.values())[0]), f"Unable to parse FW: {fw}") + codes |= result + + # Toyota places the ECU part number in their FW versions, assert all parsable + # Note that there is only one unique part number per ECU across the fleet, so this + # is not important for identification, just a sanity check. + self.assertTrue(all({code.count(b"-") > 1 for code in codes}), + f"FW does not have part number: {fw} {codes}") def test_platform_codes_spot_check(self): # Asserts basic platform code parsing behavior for a few cases From 6e5cb8c350938d78889b9a2c7d4dd048adf72559 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:52:12 -0700 Subject: [PATCH 39/65] clean up some tests --- selfdrive/car/toyota/tests/test_toyota.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 79e23ab8ef46c4..33c4891b175b9f 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -127,36 +127,14 @@ def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU excluded_platforms = { - # # CAR.LEXUS_ESH_TSS2, - # # CAR.RAV4_TSS2_2022, CAR.LEXUS_ES_TSS2, - # # CAR.RAV4_TSS2, - # # CAR.RAV4_TSS2_2023, - # CAR.RAV4_TSS2, - # # CAR.CAMRY, - # CAR.HIGHLANDER_TSS2, - # CAR.RAV4H_TSS2, CAR.LEXUS_RX_TSS2, - # # CAR.CAMRYH_TSS2, CAR.CHR, - # # CAR.RAV4H, - # CAR.RAV4H_TSS2_2022, - # # CAR.HIGHLANDERH_TSS2, - # # CAR.RAV4, - # # CAR.CHR_TSS2, - # CAR.CHRH, - # CAR.RAV4H_TSS2_2023, - # CAR.CAMRY_TSS2, CAR.COROLLA_TSS2, } platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): - # if platform != CAR.RAV4_TSS2: - # continue - # if 'RAV4 HYBRID' not in platform: - # continue - print('platform', platform) car_fw = [] for ecu, fw_versions in fw_by_addr.items(): ecu_name, addr, sub_addr = ecu @@ -166,7 +144,6 @@ def test_fuzzy_excluded_platforms(self): CP = car.CarParams.new_message(carFw=car_fw) matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) - print('matches', matches) if len(matches) == 1: self.assertEqual(list(matches)[0], platform) else: From a88eb28635055884c717947b8115385c456e69ce Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:53:17 -0700 Subject: [PATCH 40/65] clean up --- selfdrive/car/toyota/values.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 90d24b501684ec..18f7383f2900b0 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -260,7 +260,6 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: first_chunk = chunks[0] if len(first_chunk) == 8: # TODO: no part number, but some short chunks have it in subsequent chunks - # print('short version', fw) fw_match = SHORT_FW_PATTERN.search(first_chunk) if fw_match is not None: platform, major_version, sub_version = fw_match.groups() @@ -268,7 +267,6 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: # print('platform code, version', platform, major_version, sub_version) elif len(first_chunk) == 10: - # print('medium fw', fw) fw_match = MEDIUM_FW_PATTERN.search(first_chunk) if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() @@ -276,8 +274,6 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: codes[b'-'.join((part, platform, major_version))].add(sub_version) elif len(first_chunk) == 12: - # print(LONG_FW_PATTERN) - # print('long version', fw) fw_match = LONG_FW_PATTERN.search(first_chunk) if fw_match is not None: # print('got long match!') @@ -289,15 +285,9 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: - # Non-electric CAN FD platforms often do not have platform code specifiers needed - # to distinguish between hybrid and ICE. All EVs so far are either exclusively - # electric or specify electric in the platform code. - # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers - fuzzy_platform_blacklist = set() # set(CANFD_CAR - EV_CAR) candidates = set() for candidate, fws in FW_VERSIONS.items(): - # print('\n\ncandidate:', candidate) # Keep track of ECUs which pass all checks (platform codes, within sub-version range) valid_found_ecus = set() valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS} @@ -313,8 +303,6 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # Found platform codes & versions found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set())) - # print(ecu, expected_platform_codes, found_platform_codes) - # Check part number + platform code + major version matches for any found versions # Then check that sub-version for the above is within range (splits model years) for found_platform_code, found_sub_versions in found_platform_codes.items(): @@ -331,7 +319,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if valid_expected_ecus.issubset(valid_found_ecus): candidates.add(candidate) - return candidates - fuzzy_platform_blacklist + return candidates # Regex patterns for parsing more general platform-specific identifiers from FW versions. From 393bfe2479388e26ee34422ebfa1fb14051732d1 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 00:54:26 -0700 Subject: [PATCH 41/65] more clean up --- selfdrive/car/toyota/values.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 18f7383f2900b0..e3dd4b387dc153 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -264,21 +264,17 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: if fw_match is not None: platform, major_version, sub_version = fw_match.groups() codes[b'-'.join((platform, major_version))].add(sub_version) - # print('platform code, version', platform, major_version, sub_version) elif len(first_chunk) == 10: fw_match = MEDIUM_FW_PATTERN.search(first_chunk) if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() - # print(part, platform, major_version, sub_version) codes[b'-'.join((part, platform, major_version))].add(sub_version) elif len(first_chunk) == 12: fw_match = LONG_FW_PATTERN.search(first_chunk) if fw_match is not None: - # print('got long match!') part, platform, major_version, sub_version = fw_match.groups() - # print(first_chunk, fw_match, fw_match.groups()) codes[b'-'.join((part, platform, major_version))].add(sub_version) return dict(codes) From cde385af6e8b8af46afd105fabdc0d28651903a4 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 01:06:15 -0700 Subject: [PATCH 42/65] static --- selfdrive/car/toyota/tests/test_toyota.py | 4 ++-- selfdrive/car/toyota/values.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 33c4891b175b9f..5377b7d96cff81 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -3,7 +3,7 @@ import unittest from cereal import car -from selfdrive.car.fw_versions import build_fw_dict +from openpilot.selfdrive.car.fw_versions import build_fw_dict from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, get_platform_codes @@ -90,7 +90,7 @@ def test_fw_format(self): # Toyota places the ECU part number in their FW versions, assert all parsable # Note that there is only one unique part number per ECU across the fleet, so this # is not important for identification, just a sanity check. - self.assertTrue(all({code.count(b"-") > 1 for code in codes}), + self.assertTrue(all(code.count(b"-") > 1 for code in codes), f"FW does not have part number: {fw} {codes}") def test_platform_codes_spot_check(self): diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index e3dd4b387dc153..f08c85fb4f391a 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -2,7 +2,7 @@ from collections import defaultdict from dataclasses import dataclass, field from enum import Enum, IntFlag -from typing import Dict, List, Set, Tuple, Union +from typing import Dict, List, Set, Union from cereal import car from openpilot.common.conversions import Conversions as CV From 93e8f799386aaa6f9862f76a05d2cb537b56d618 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 01:16:16 -0700 Subject: [PATCH 43/65] this is all it took? --- selfdrive/car/toyota/tests/print_platform_codes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index 25b921819df857..11a5d5b754a6ff 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -7,7 +7,7 @@ ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} if __name__ == "__main__": - parts_for_ecu = defaultdict(set) + parts_for_ecu: dict = defaultdict(set) for car_model, ecus in FW_VERSIONS.items(): print() From 3a59e998404400f7eb44ec97121b1fa68606a12c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 01:17:42 -0700 Subject: [PATCH 44/65] add note --- selfdrive/car/toyota/tests/test_toyota.py | 1 + 1 file changed, 1 insertion(+) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 5377b7d96cff81..ca6a7b1a171f33 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -126,6 +126,7 @@ def test_platform_codes_spot_check(self): def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU + # Note: this list is so small because other ECUs are not shared between hybrid & ICE, commonly the ABS. excluded_platforms = { CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, From d1e3d1808bbde161ccab3896096bfa4c45f71f0d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 18 Sep 2023 01:17:50 -0700 Subject: [PATCH 45/65] ... --- selfdrive/car/toyota/tests/test_toyota.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index ca6a7b1a171f33..aa8e4029bbfb0f 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -125,8 +125,8 @@ def test_platform_codes_spot_check(self): def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. - # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU - # Note: this list is so small because other ECUs are not shared between hybrid & ICE, commonly the ABS. + # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU. + # Note: this list is so small because other ECUs are not shared between hybrid & ICE, commonly the ABS excluded_platforms = { CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, From c34024ad3dd5033f12de2fdcd6d80c60294940f5 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 20 Sep 2023 03:16:01 -0700 Subject: [PATCH 46/65] use less ECUs --- selfdrive/car/toyota/tests/test_toyota.py | 13 +++++++++ selfdrive/car/toyota/values.py | 32 +++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index aa8e4029bbfb0f..94fd1b3602305d 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import random from hypothesis import given, settings, strategies as st import unittest @@ -131,8 +132,18 @@ def test_fuzzy_excluded_platforms(self): CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, CAR.CHR, + CAR.CHRH, CAR.COROLLA_TSS2, + CAR.COROLLAH_TSS2, + CAR.RAV4_TSS2, + CAR.RAV4H_TSS2, + CAR.LEXUS_NX_TSS2, + CAR.LEXUS_NXH, + CAR.LEXUS_NXH_TSS2, + CAR.LEXUS_NX, + CAR.LEXUS_RXH_TSS2, } + # excluded_platforms = {_car for _car in FW_VERSIONS} platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): @@ -148,8 +159,10 @@ def test_fuzzy_excluded_platforms(self): if len(matches) == 1: self.assertEqual(list(matches)[0], platform) else: + print('matches', platform, matches) platforms_with_shared_codes.add(platform) + print(len(platforms_with_shared_codes), len(excluded_platforms)) self.assertEqual(platforms_with_shared_codes, excluded_platforms, (len(platforms_with_shared_codes), len(FW_VERSIONS))) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 4379d9d622e7a7..a760a2549f3a82 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -301,17 +301,27 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # Check part number + platform code + major version matches for any found versions # Then check that sub-version for the above is within range (splits model years) - for found_platform_code, found_sub_versions in found_platform_codes.items(): - if found_platform_code in expected_platform_codes: - expected_sub_versions = expected_platform_codes[found_platform_code] - # Check any sub-version within range in the database for this part-platform-version combo - if any(min(expected_sub_versions) <= found_sub_version <= max(expected_sub_versions) for - found_sub_version in found_sub_versions): - valid_found_ecus.add(addr) - break + # Sub-versions are incremented for minor recalls, do not need to be checked. + # Platform codes and major versions change for different physical parts. For example, + # engine's major version describes actual Toyota engine type. + + if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): + break + valid_found_ecus.add(addr) + + # for found_platform_code, found_sub_versions in found_platform_codes.items(): + # if found_platform_code in expected_platform_codes: + # expected_sub_versions = expected_platform_codes[found_platform_code] + # + # # Check any sub-version within range in the database for this part-platform-version combo + # if any(min(expected_sub_versions) <= found_sub_version <= max(expected_sub_versions) for + # found_sub_version in found_sub_versions): + # valid_found_ecus.add(addr) + # break # If all live ECUs pass all checks for candidate, add it as a match + # print(candidate, valid_expected_ecus, valid_found_ecus) if valid_expected_ecus.issubset(valid_found_ecus): candidates.add(candidate) @@ -333,9 +343,9 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: FW_LEN_CODE = re.compile(b'^[\x01-\x03]') # highest seen is 3 chunks, 16 bytes each FW_CHUNK_LEN = 16 -# List of ECUs expected to have platform codes -# Engine is not well understood at this time (lots of variability in major version) -PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] +# List of ECUs that are most unique across openpilot platforms +# TODO: use hybrid ECU, splits similar ICE and hybrid variants +PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.eps, Ecu.fwdCamera] # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. From 0e629e3f079b680236cc02386946560647ab1e4d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 20 Sep 2023 17:20:35 -0700 Subject: [PATCH 47/65] bump --- cereal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cereal b/cereal index 4b334f6f10877e..95485f268cacb1 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 4b334f6f10877e4a666b23983de2d27934ebf3b1 +Subproject commit 95485f268cacb16186163744224c9dd8644ecd49 From 368b2c4f21d370c2ff203d97e4568b1ca2dd71aa Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 23 Sep 2023 02:25:08 -0700 Subject: [PATCH 48/65] todo --- selfdrive/car/toyota/tests/test_toyota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index ca3f3aa9fddadc..20434fc2f6c164 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -126,6 +126,7 @@ def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU. # Note: this list is so small because other ECUs are not shared between hybrid & ICE, commonly the ABS + # TODO: should this be put into the fuzzy FP function as a blacklist? excluded_platforms = { CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, @@ -141,7 +142,6 @@ def test_fuzzy_excluded_platforms(self): CAR.LEXUS_NX, CAR.LEXUS_RXH_TSS2, } - # excluded_platforms = {_car for _car in FW_VERSIONS} platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): From 7faf0484c87a5bb6ecadd3426f2a294a3355bfee Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 23 Sep 2023 02:29:16 -0700 Subject: [PATCH 49/65] clean up fuzzy fp function --- selfdrive/car/toyota/values.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index a6acb6e6d3b19e..3c52ce548a7abb 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -300,25 +300,12 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set())) # Check part number + platform code + major version matches for any found versions - # Then check that sub-version for the above is within range (splits model years) - + # Platform codes and major versions change for different physical parts, generation, API, etc. # Sub-versions are incremented for minor recalls, do not need to be checked. - # Platform codes and major versions change for different physical parts. For example, - # engine's major version describes actual Toyota engine type. - if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): break - valid_found_ecus.add(addr) - # for found_platform_code, found_sub_versions in found_platform_codes.items(): - # if found_platform_code in expected_platform_codes: - # expected_sub_versions = expected_platform_codes[found_platform_code] - # - # # Check any sub-version within range in the database for this part-platform-version combo - # if any(min(expected_sub_versions) <= found_sub_version <= max(expected_sub_versions) for - # found_sub_version in found_sub_versions): - # valid_found_ecus.add(addr) - # break + valid_found_ecus.add(addr) # If all live ECUs pass all checks for candidate, add it as a match # print(candidate, valid_expected_ecus, valid_found_ecus) From a45836f18bbeaf3d8bb49dabc4925cd19f1d8061 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 23 Sep 2023 02:29:44 -0700 Subject: [PATCH 50/65] make deterministic in a feat of engineering --- selfdrive/car/toyota/tests/test_toyota.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 20434fc2f6c164..49d88d642a29d0 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import random from hypothesis import given, settings, strategies as st import unittest From 2b520a6e4ecbd12f5a5f41066e60c0d3eb8ab58b Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 23 Sep 2023 23:48:12 -0700 Subject: [PATCH 51/65] add temp exclude_fw argument for testing --- selfdrive/car/toyota/values.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 3c52ce548a7abb..1858b25bf2c7d2 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,7 +280,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes]) -> Set[str]: candidates = set() for candidate, fws in FW_VERSIONS.items(): @@ -293,6 +293,9 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if ecu[0] not in PLATFORM_CODE_ECUS: continue + # ignore FW if in exclude dict and there's more than one fw + expected_versions = [fw for fw in expected_versions if fw != exclude_fw[ecu[0]] and len(expected_versions) > 1] + # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 2d9a4ad5b276757ca30aeb31936a4d7b6e7a6cad Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 24 Sep 2023 00:01:17 -0700 Subject: [PATCH 52/65] fix logic --- selfdrive/car/toyota/values.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 1858b25bf2c7d2..89e2d0728fe039 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,8 +280,10 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes]) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: candidates = set() + if exclude_fw is None: + exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -294,7 +296,8 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes]) -> Set continue # ignore FW if in exclude dict and there's more than one fw - expected_versions = [fw for fw in expected_versions if fw != exclude_fw[ecu[0]] and len(expected_versions) > 1] + expected_versions = [fw for fw in expected_versions if ecu[0] not in exclude_fw or + (fw != exclude_fw[ecu[0]] or len(expected_versions) == 1)] # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 81c107652f86950022f1f145ab5f229e663521bb Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 24 Sep 2023 00:01:30 -0700 Subject: [PATCH 53/65] add blacklist --- selfdrive/car/toyota/tests/test_toyota.py | 26 ++++------------------- selfdrive/car/toyota/values.py | 7 +++++- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 49d88d642a29d0..c29eae870d9578 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -5,7 +5,8 @@ from cereal import car from openpilot.selfdrive.car.fw_versions import build_fw_dict from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ - FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, get_platform_codes + FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, FUZZY_EXCLUDED_PLATFORMS, \ + get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -123,25 +124,6 @@ def test_platform_codes_spot_check(self): def test_fuzzy_excluded_platforms(self): # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. - # This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU. - # Note: this list is so small because other ECUs are not shared between hybrid & ICE, commonly the ABS - # TODO: should this be put into the fuzzy FP function as a blacklist? - excluded_platforms = { - CAR.LEXUS_ES_TSS2, - CAR.LEXUS_RX_TSS2, - CAR.CHR, - CAR.CHRH, - CAR.COROLLA_TSS2, - CAR.COROLLAH_TSS2, - CAR.RAV4_TSS2, - CAR.RAV4H_TSS2, - CAR.LEXUS_NX_TSS2, - CAR.LEXUS_NXH, - CAR.LEXUS_NXH_TSS2, - CAR.LEXUS_NX, - CAR.LEXUS_RXH_TSS2, - } - platforms_with_shared_codes = set() for platform, fw_by_addr in FW_VERSIONS.items(): car_fw = [] @@ -159,8 +141,8 @@ def test_fuzzy_excluded_platforms(self): print('matches', platform, matches) platforms_with_shared_codes.add(platform) - print(len(platforms_with_shared_codes), len(excluded_platforms)) - self.assertEqual(platforms_with_shared_codes, excluded_platforms, (len(platforms_with_shared_codes), len(FW_VERSIONS))) + print(len(platforms_with_shared_codes), len(FUZZY_EXCLUDED_PLATFORMS)) + self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS))) if __name__ == "__main__": diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 89e2d0728fe039..a5e92f2d213d39 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -318,7 +318,7 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) if valid_expected_ecus.issubset(valid_found_ecus): candidates.add(candidate) - return candidates + return candidates - FUZZY_EXCLUDED_PLATFORMS # Regex patterns for parsing more general platform-specific identifiers from FW versions. @@ -348,6 +348,11 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) # - eps: describes lateral API changes for the EPS, such as using LTA for lane keeping and rejecting LKA messages PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps] +# These platforms have at least one platform code for all ECUs shared with another platform. +# This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU +FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, CAR.CHR, CAR.CHRH, CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2, + CAR.RAV4_TSS2, CAR.RAV4H_TSS2, CAR.LEXUS_NX_TSS2, CAR.LEXUS_NXH, CAR.LEXUS_NXH_TSS2, + CAR.LEXUS_NX, CAR.LEXUS_RXH_TSS2} # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. # Toyota diagnostic software first gets the supported data ids, then queries them one by one. From 4e4555ed060706c0d64b7d2ed38dd150c1f246cd Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 24 Sep 2023 00:05:34 -0700 Subject: [PATCH 54/65] add platform and its matches --- selfdrive/car/toyota/tests/test_toyota.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index c29eae870d9578..67c9f0d92c2f50 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -138,8 +138,9 @@ def test_fuzzy_excluded_platforms(self): if len(matches) == 1: self.assertEqual(list(matches)[0], platform) else: + # If a platform has multiple matches, add it and its matches print('matches', platform, matches) - platforms_with_shared_codes.add(platform) + platforms_with_shared_codes |= {platform, *matches} print(len(platforms_with_shared_codes), len(FUZZY_EXCLUDED_PLATFORMS)) self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS))) From 27aac16187aaad71eae5ce75601d147521d539f3 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sun, 24 Sep 2023 00:20:42 -0700 Subject: [PATCH 55/65] fix for nb --- selfdrive/car/toyota/values.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index a5e92f2d213d39..1ee9fe1f9d3b23 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -296,8 +296,8 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) continue # ignore FW if in exclude dict and there's more than one fw - expected_versions = [fw for fw in expected_versions if ecu[0] not in exclude_fw or - (fw != exclude_fw[ecu[0]] or len(expected_versions) == 1)] + expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or + (fw != exclude_fw[addr] or len(expected_versions) == 1)] # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 0e3b47c5a30ebdb647caf5d792b10f517cbdf48f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 27 Sep 2023 21:47:43 -0700 Subject: [PATCH 56/65] remove fw exclusion --- selfdrive/car/toyota/values.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 6c28a4d9070937..62502096259945 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,10 +280,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: candidates = set() - if exclude_fw is None: - exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -295,10 +293,6 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) if ecu[0] not in PLATFORM_CODE_ECUS: continue - # ignore FW if in exclude dict and there's more than one fw - expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or - (fw != exclude_fw[addr] or len(expected_versions) == 1)] - # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 42c55f006a4a0596ff404c67db1a88eb64b45d9e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 27 Sep 2023 22:08:57 -0700 Subject: [PATCH 57/65] Revert "remove fw exclusion" This reverts commit 0e3b47c5a30ebdb647caf5d792b10f517cbdf48f. --- selfdrive/car/toyota/values.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 62502096259945..6c28a4d9070937 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,8 +280,10 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: candidates = set() + if exclude_fw is None: + exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -293,6 +295,10 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if ecu[0] not in PLATFORM_CODE_ECUS: continue + # ignore FW if in exclude dict and there's more than one fw + expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or + (fw != exclude_fw[addr] or len(expected_versions) == 1)] + # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 3aeea9083657ff1678305d15ab5b2bf1b35d85a0 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 27 Sep 2023 23:29:48 -0700 Subject: [PATCH 58/65] clean up --- selfdrive/car/toyota/tests/test_toyota.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 67c9f0d92c2f50..00312a59bd6464 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -139,10 +139,8 @@ def test_fuzzy_excluded_platforms(self): self.assertEqual(list(matches)[0], platform) else: # If a platform has multiple matches, add it and its matches - print('matches', platform, matches) platforms_with_shared_codes |= {platform, *matches} - print(len(platforms_with_shared_codes), len(FUZZY_EXCLUDED_PLATFORMS)) self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS))) From 0f874233422d2007cc87bfb1c714cabed6e80049 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 27 Sep 2023 23:29:55 -0700 Subject: [PATCH 59/65] Revert "Revert "remove fw exclusion"" This reverts commit 42c55f006a4a0596ff404c67db1a88eb64b45d9e. --- selfdrive/car/toyota/values.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 6c28a4d9070937..62502096259945 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -280,10 +280,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: candidates = set() - if exclude_fw is None: - exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -295,10 +293,6 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) if ecu[0] not in PLATFORM_CODE_ECUS: continue - # ignore FW if in exclude dict and there's more than one fw - expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or - (fw != exclude_fw[addr] or len(expected_versions) == 1)] - # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 3490034f330fae9e426ccff5c19b53573724e70f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 01:29:11 -0700 Subject: [PATCH 60/65] these two have similar chassis according to wikipedia (but mass is relatively different) --- selfdrive/car/toyota/values.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 8d74b49ff822ad..ed5c7da68630d2 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -347,10 +347,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps] # These platforms have at least one platform code for all ECUs shared with another platform. -# This list can be shrunk as we combine platforms, detect features, and add the hybrid ECU -FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2, CAR.CHR, CAR.CHRH, CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2, - CAR.RAV4_TSS2, CAR.RAV4H_TSS2, CAR.LEXUS_NX_TSS2, CAR.LEXUS_NXH, CAR.LEXUS_NXH_TSS2, - CAR.LEXUS_NX, CAR.LEXUS_RXH_TSS2} +FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2} # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. # Toyota diagnostic software first gets the supported data ids, then queries them one by one. From 2411967f5a2b0712773d614ee589321cca4d52a5 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 01:57:53 -0700 Subject: [PATCH 61/65] Revert "Revert "Revert "remove fw exclusion""" This reverts commit 0f874233422d2007cc87bfb1c714cabed6e80049. --- selfdrive/car/toyota/values.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index ed5c7da68630d2..7c0a9926e7e2c3 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -284,8 +284,10 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: candidates = set() + if exclude_fw is None: + exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -297,6 +299,10 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if ecu[0] not in PLATFORM_CODE_ECUS: continue + # ignore FW if in exclude dict and there's more than one fw + expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or + (fw != exclude_fw[addr] or len(expected_versions) == 1)] + # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From 775c0972a27d36aedcba8e68ba2999cb4a09bc1d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 06:28:45 -0700 Subject: [PATCH 62/65] Revert "Revert "Revert "Revert "remove fw exclusion"""" This reverts commit 2411967f5a2b0712773d614ee589321cca4d52a5. --- selfdrive/car/toyota/values.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 7c0a9926e7e2c3..ed5c7da68630d2 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -284,10 +284,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) -def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) -> Set[str]: +def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: candidates = set() - if exclude_fw is None: - exclude_fw = dict() for candidate, fws in FW_VERSIONS.items(): # Keep track of ECUs which pass all checks (platform codes, within sub-version range) @@ -299,10 +297,6 @@ def match_fw_to_car_fuzzy(live_fw_versions, exclude_fw: dict[int, bytes] = None) if ecu[0] not in PLATFORM_CODE_ECUS: continue - # ignore FW if in exclude dict and there's more than one fw - expected_versions = [fw for fw in expected_versions if addr not in exclude_fw or - (fw != exclude_fw[addr] or len(expected_versions) == 1)] - # Expected platform codes & versions expected_platform_codes = get_platform_codes(expected_versions) From d9ebf5dfa0780849353b37ba0fca37a4840faf94 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 06:41:04 -0700 Subject: [PATCH 63/65] oof --- selfdrive/car/toyota/values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index ed5c7da68630d2..e0483f5200a725 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -316,7 +316,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: if valid_expected_ecus.issubset(valid_found_ecus): candidates.add(candidate) - return candidates - FUZZY_EXCLUDED_PLATFORMS + return {str(c) for c in (candidates - FUZZY_EXCLUDED_PLATFORMS)} # Regex patterns for parsing more general platform-specific identifiers from FW versions. From 63b9e53340023d22ddf883876f91acf3ce649a66 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 06:41:16 -0700 Subject: [PATCH 64/65] shadows global variable --- selfdrive/car/hyundai/values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/car/hyundai/values.py b/selfdrive/car/hyundai/values.py index 5489d8cffabb59..110f0c540ad9ba 100644 --- a/selfdrive/car/hyundai/values.py +++ b/selfdrive/car/hyundai/values.py @@ -400,7 +400,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # to distinguish between hybrid and ICE. All EVs so far are either exclusively # electric or specify electric in the platform code. # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers - fuzzy_platform_blacklist = {str(car) for car in set(CANFD_CAR - EV_CAR)} + fuzzy_platform_blacklist = {str(c) for c in set(CANFD_CAR - EV_CAR)} candidates: Set[str] = set() for candidate, fws in FW_VERSIONS.items(): From 2f87fb8978f1082c4b1bd989dbb1adb13671fd75 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 18:08:58 -0700 Subject: [PATCH 65/65] rm comment --- selfdrive/car/toyota/values.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 4a6513e7487978..7d26ffd2e78df5 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -312,7 +312,6 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: valid_found_ecus.add(addr) # If all live ECUs pass all checks for candidate, add it as a match - # print(candidate, valid_expected_ecus, valid_found_ecus) if valid_expected_ecus.issubset(valid_found_ecus): candidates.add(candidate)