From b43d406fe6112d21b1bfa028eb191e9b11255dd7 Mon Sep 17 00:00:00 2001 From: ahatamizadeh Date: Mon, 7 Mar 2022 09:14:26 -0800 Subject: [PATCH 1/2] update Transchex model Signed-off-by: ahatamizadeh --- monai/networks/nets/transchex.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/monai/networks/nets/transchex.py b/monai/networks/nets/transchex.py index cb9baa5e65..b03ff5a17d 100644 --- a/monai/networks/nets/transchex.py +++ b/monai/networks/nets/transchex.py @@ -185,12 +185,15 @@ class BertMixedLayer(nn.Module): def __init__(self, config) -> None: super().__init__() - self.att = BertAttention(config) - self.output = BertOutput(config) + self.att_x = BertAttention(config) + self.output_x = BertOutput(config) + self.att_y = BertAttention(config) + self.output_y = BertOutput(config) def forward(self, x, y): - output = self.att(x, y) - return self.output(output, x) + output_x = self.att_x(x, y) + output_y = self.att_y(y, x) + return self.output_x(output_x, x), self.output_y(output_y, y) class Pooler(nn.Module): @@ -237,12 +240,12 @@ def __init__( def forward(self, input_ids, token_type_ids=None, vision_feats=None, attention_mask=None): language_features = self.embeddings(input_ids, token_type_ids) for layer in self.vision_encoder: - hidden_state_vision = layer(vision_feats, None)[0] + vision_feats = layer(vision_feats, None)[0] for layer in self.language_encoder: - hidden_state_language = layer(language_features, attention_mask)[0] + language_features = layer(language_features, attention_mask)[0] for layer in self.mixed_encoder: - hidden_state_mixed = layer(hidden_state_language, hidden_state_vision) - return hidden_state_mixed + language_features, vision_feats = layer(language_features, vision_feats) + return language_features, vision_feats class Transchex(torch.nn.Module): @@ -367,9 +370,9 @@ def forward(self, input_ids, token_type_ids=None, vision_feats=None): vision_feats = self.vision_proj(vision_feats).flatten(2).transpose(1, 2) vision_feats = self.norm_vision_pos(vision_feats) vision_feats = vision_feats + self.pos_embed_vis - hidden_state_mixed = self.multimodal( + hidden_state_lang, hidden_state_vis = self.multimodal( input_ids=input_ids, token_type_ids=token_type_ids, vision_feats=vision_feats, attention_mask=attention_mask ) - pooled_features = self.pooler(hidden_state_mixed) + pooled_features = self.pooler(hidden_state_lang) logits = self.cls_head(self.drop(pooled_features)) return logits From 0683a58daae4a3f2f750f13999d7c759839dea96 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Mon, 7 Mar 2022 18:53:43 +0000 Subject: [PATCH 2/2] fixes test case downloading issue Signed-off-by: Wenqi Li --- tests/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index c186ca1675..0f22f65d33 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -114,6 +114,8 @@ def skip_if_downloading_fails(): except (ContentTooShortError, HTTPError, ConnectionError) as e: raise unittest.SkipTest(f"error while downloading: {e}") from e except RuntimeError as rt_e: + if "unexpected EOF" in str(rt_e): + raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e # incomplete download if "network issue" in str(rt_e): raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e if "gdown dependency" in str(rt_e): # no gdown installed