Merge branch 'develop' into feat/DEVSU-2616-create-project-when-submitting-a-report-with-report_async

Author: sshugsc

README.md

@@ -15,6 +15,7 @@ For documentation on how to create reports using the IPR adaptor, see the [main
 
 - [Getting Started](#getting-started)
   - [Install (For developers)](#install-for-developers)
+  - [JSON Validate and Upload to IPR](#json-validate-and-upload-to-ipr)
 - [Documentation](#documentation)
 - [Deployment (Publishing)](#deployment-publishing)
 
@@ -82,6 +83,17 @@ export DELETE_UPLOAD_TEST_REPORTS=0
 pytest tests
 ```
 
+### JSON Validate and Upload to IPR
+If you only want to validate the json content, use
+```bash
+ipr --password $IPR_PASS -c 'path/to/content.json' --validate_json
+```
+
+If you only want to upload the json directly to ipr and skip all the preprocessing, use
+```bash
+ipr --password $IPR_PASS -c 'path/to/content.json' --upload_json
+```
+
 ## Documentation
 
 The user documentation for this tool is hosted with the [main documentation site](https://bcgsc.github.io/pori/).

pori_python/ipr/connection.py

@@ -95,7 +95,11 @@ def delete(self, uri: str, data: Dict = {}, **kwargs) -> Dict:
         )
 
     def upload_report(
-        self, content: Dict, mins_to_wait: int = 5, async_upload: bool = False
+        self,
+        content: Dict,
+        mins_to_wait: int = 5,
+        async_upload: bool = False,
+        ignore_extra_fields: bool = False,
     ) -> Dict:
         if async_upload:
             # if async is used, the response for reports-async contains either 'jobStatus'
@@ -115,7 +119,10 @@ def upload_report(
                 except Exception as err:
                     raise Exception(f"Project creation failed due to {err}")
 
-            initial_result = self.post("reports-async", content)
+            if ignore_extra_fields:
+                initial_result = self.post("reports-async?ignore_extra_fields=true", content)
+            else:
+                initial_result = self.post("reports-async", content)
 
             report_id = initial_result["ident"]
 
@@ -172,7 +179,10 @@ def check_status(interval: int = 5, num_attempts: int = 5):
 
             return current_status
         else:
-            return self.post("reports", content)
+            if ignore_extra_fields:
+                return self.post("reports?ignore_extra_fields=true", content)
+            else:
+                return self.post("reports", content)
 
     def set_analyst_comments(self, report_id: str, data: Dict) -> Dict:
         """
@@ -226,3 +236,11 @@ def get_spec(self) -> Dict:
         Get the current IPR spec, for the purposes of current report upload fields
         """
         return self.request("/spec.json", method="GET")
+
+    def validate_json(self, content: Dict) -> Dict:
+        """
+        Validate the provided json schema
+        """
+        result = self.post("reports/schema", content)
+        logger.info(f"{result['message']}")
+        return result

pori_python/ipr/main.py

@@ -108,6 +108,7 @@ def command_interface() -> None:
     parser.add_argument(
         "-o",
         "--output_json_path",
+        default=f"pori_python_report_{timestamp()}.json",
         help="path to a JSON to output the report upload body",
     )
     parser.add_argument(
@@ -134,6 +135,24 @@ def command_interface() -> None:
         action="store_true",
         help="True to include matches to multivariant statements where not all variants are present",
     )
+    parser.add_argument(
+        "--upload_json",
+        default=False,
+        action="store_true",
+        help="True to skip all the preprocessing and just submit a json to ipr",
+    )
+    parser.add_argument(
+        "--validate_json",
+        default=False,
+        action="store_true",
+        help="True if only need to validate the json",
+    )
+    parser.add_argument(
+        "--ignore_extra_fields",
+        default=False,
+        action="store_true",
+        help="True if ignore extra fields in json",
+    )
     args = parser.parse_args()
 
     with open(args.content, "r") as fh:
@@ -155,6 +174,9 @@ def command_interface() -> None:
         async_upload=args.async_upload,
         mins_to_wait=args.mins_to_wait,
         allow_partial_matches=args.allow_partial_matches,
+        upload_json=args.upload_json,
+        validate_json=args.validate_json,
+        ignore_extra_fields=args.ignore_extra_fields,
     )
 
 
@@ -288,6 +310,9 @@ def ipr_report(
     include_nonspecific_project: bool = False,
     include_nonspecific_template: bool = False,
     allow_partial_matches: bool = False,
+    upload_json: bool = False,
+    validate_json: bool = False,
+    ignore_extra_fields: bool = False,
     tmb_high: float = TMB_SIGNATURE_HIGH_THRESHOLD,
 ) -> Dict:
     """Run the matching and create the report JSON for upload to IPR.
@@ -327,6 +352,20 @@ def ipr_report(
         format="%(asctime)s %(name)s %(levelname)s %(message)s",
         datefmt="%m-%d-%y %H:%M:%S",
     )
+
+    # IPR CONNECTION
+    ipr_conn = IprConnection(username, password, ipr_url)
+
+    if validate_json:
+        ipr_result = ipr_conn.validate_json(content)
+        return ipr_result
+
+    if upload_json:
+        ipr_result = ipr_conn.upload_report(
+            content, mins_to_wait, async_upload, ignore_extra_fields
+        )
+        return ipr_result
+
     # validate the JSON content follows the specification
     try:
         validate_report_content(content)
@@ -365,10 +404,6 @@ def ipr_report(
         small_mutations, expression_variants, copy_variants, structural_variants
     )
 
-    # IPR CONNECTION
-    ipr_conn = IprConnection(username, password, ipr_url)
-    ipr_spec = ipr_conn.get_spec()
-
     # GKB CONNECTION
     if graphkb_url:
         logger.info(f"connecting to graphkb: {graphkb_url}")
@@ -509,6 +544,7 @@ def ipr_report(
     )
     output.setdefault("images", []).extend(select_expression_plots(gkb_matches, all_variants))
 
+    ipr_spec = ipr_conn.get_spec()
     output = clean_unsupported_content(output, ipr_spec)
     ipr_result = None
     upload_error = None
@@ -517,19 +553,20 @@ def ipr_report(
     if ipr_upload:
         try:
             logger.info(f"Uploading to IPR {ipr_conn.url}")
-            ipr_result = ipr_conn.upload_report(output, mins_to_wait, async_upload)
+            ipr_result = ipr_conn.upload_report(
+                output, mins_to_wait, async_upload, ignore_extra_fields
+            )
             logger.info(ipr_result)
             output.update(ipr_result)
         except Exception as err:
             upload_error = err
             logger.error(f"ipr_conn.upload_report failed: {err}", exc_info=True)
 
     # SAVE TO JSON FILE
-    if output_json_path:
-        if always_write_output_json or not ipr_result:
-            logger.info(f"Writing IPR upload json to: {output_json_path}")
-            with open(output_json_path, "w") as fh:
-                fh.write(json.dumps(output))
+    if always_write_output_json:
+        logger.info(f"Writing IPR upload json to: {output_json_path}")
+        with open(output_json_path, "w") as fh:
+            fh.write(json.dumps(output))
 
     logger.info(f"made {graphkb_conn.request_count} requests to graphkb")
     logger.info(f"average load {int(graphkb_conn.load or 0)} req/s")