Support --continue/-c in the metaschema command

completion/jsonschema.bash

@@ -112,7 +112,7 @@ _jsonschema() {
       fi
       ;;
     metaschema)
-      local options="--extension -e --ignore -i --trace -t"
+      local options="--extension -e --ignore -i --trace -t --continue -c"
       if [[ ${current} == -* ]]
       then
         COMPREPLY=( $(compgen -W "${options} ${global_options}" -- "${current}") )

completion/jsonschema.zsh

@@ -68,6 +68,7 @@ _jsonschema() {
             '(--extension -e)'{--extension,-e}'[Specify file extension]:extension:_jsonschema_extensions' \
             '(--ignore -i)'{--ignore,-i}'[Ignore schemas or directories]:path:_files' \
             '(--trace -t)'{--trace,-t}'[Enable trace output]' \
+            '(--continue -c)'{--continue,-c}'[Report every failing schema]' \
             '*:schema file:_files -g "*.json *.yaml *.yml"'
           ;;
         compile)

docs/metaschema.markdown

@@ -10,7 +10,8 @@ jsonschema metaschema [schemas-or-directories...]
   [--header/-H "<name>: <value>"]
   [--resolve/-r <schemas-or-directories> ...]
   [--ignore/-i <schemas-or-directories>] [--trace/-t]
-  [--default-dialect/-d <uri>] [--json/-j] [--format-assertion/-F]
+  [--default-dialect/-d <uri>] [--json/-j] [--continue/-c]
+  [--format-assertion/-F]
   [--configuration/-C <path>]
 ```
 

src/command_metaschema.cc

@@ -10,6 +10,7 @@
 
 #include <cassert>     // assert
 #include <iostream>    // std::cout, std::cerr
+#include <iterator>    // std::next
 #include <map>         // std::map
 #include <sstream>     // std::ostringstream
 #include <string>      // std::string
@@ -54,13 +55,18 @@ auto sourcemeta::jsonschema::metaschema(
   validate_http_headers(options);
   const auto trace{options.contains("trace")};
   const auto json_output{options.contains("json")};
+  const auto continue_on_error{options.contains("continue")};
 
   ValidationSummary summary;
   sourcemeta::blaze::Evaluator evaluator;
 
   std::map<std::string, sourcemeta::blaze::Template> cache;
 
-  for (const auto &entry : for_each_json(options, InputRequirement::NonEmpty)) {
+  const auto entries{for_each_json(options, InputRequirement::NonEmpty)};
+  for (auto iterator{entries.cbegin()}; iterator != entries.cend();
+       ++iterator) {
+    const auto &entry{*iterator};
+    const auto failures_before{summary.failed};
     summary.validated += 1;
     if (!entry.second.is_object() && !entry.second.is_boolean()) {
       throw NotSchemaError{entry.from_stdin ? stdin_path()
@@ -199,12 +205,22 @@ auto sourcemeta::jsonschema::metaschema(
           sourcemeta::blaze::SchemaAnchorCollisionError>(entry.resolution_base,
                                                          error);
     }
+
+    if (summary.failed > failures_before && !continue_on_error) {
+      summary.stopped = std::next(iterator) != entries.cend();
+      break;
+    }
   }
 
   if (!json_output && !trace) {
     print_summary(summary, options, std::cerr);
   }
 
+  if (summary.stopped) {
+    LOG_WARNING()
+        << "Stopped at first failure, pass --continue/-c to keep going\n";
+  }
+
   if (summary.failed > 0) {
     throw Fail{EXIT_EXPECTED_FAILURE};
   }

src/main.cc

@@ -60,7 +60,7 @@ Global Options:
 
    metaschema [schemas-or-directories...] [--extension/-e <extension>]
               [--ignore/-i <schemas-or-directories>] [--trace/-t]
-              [--format-assertion/-F]
+              [--continue/-c] [--format-assertion/-F]
 
        Validate that a schema or a set of schemas are valid with respect
        to their metaschemas.
@@ -227,6 +227,7 @@ auto jsonschema_main(const std::string &program, const std::string &command,
   if (command == "metaschema") {
     app.flag("trace", {"t"});
     app.flag("format-assertion", {"F"});
+    app.flag("continue", {"c"});
     app.option("extension", {"e"});
     app.option("ignore", {"i"});
     app.parse(argc, argv, {.skip = 1});

test/CMakeLists.txt

@@ -375,6 +375,10 @@ add_jsonschema_test(validate/pass_bundled_metaschema)
 add_jsonschema_test(metaschema/pass_trace)
 add_jsonschema_test(metaschema/fail_trace)
 add_jsonschema_test(metaschema/fail_directory)
+add_jsonschema_test(metaschema/fail_directory_stop)
+add_jsonschema_test(metaschema/fail_directory_stop_json)
+add_jsonschema_test(metaschema/fail_directory_continue)
+add_jsonschema_test(metaschema/fail_directory_continue_json)
 add_jsonschema_test(metaschema/fail_no_schemas_directory)
 add_jsonschema_test(metaschema/fail_no_schemas_directory_json)
 add_jsonschema_test(metaschema/fail_no_schemas_extension)

test/help_command.clitest

@@ -53,7 +53,7 @@ WRITE expected.txt UNTIL EOF
 1>
 1>    metaschema [schemas-or-directories...] [--extension/-e <extension>]
 1>               [--ignore/-i <schemas-or-directories>] [--trace/-t]
-1>               [--format-assertion/-F]
+1>               [--continue/-c] [--format-assertion/-F]
 1>
 1>        Validate that a schema or a set of schemas are valid with respect
 1>        to their metaschemas.

test/help_option_long.clitest

@@ -53,7 +53,7 @@ WRITE expected.txt UNTIL EOF
 1>
 1>    metaschema [schemas-or-directories...] [--extension/-e <extension>]
 1>               [--ignore/-i <schemas-or-directories>] [--trace/-t]
-1>               [--format-assertion/-F]
+1>               [--continue/-c] [--format-assertion/-F]
 1>
 1>        Validate that a schema or a set of schemas are valid with respect
 1>        to their metaschemas.

test/help_option_short.clitest

@@ -53,7 +53,7 @@ WRITE expected.txt UNTIL EOF
 1>
 1>    metaschema [schemas-or-directories...] [--extension/-e <extension>]
 1>               [--ignore/-i <schemas-or-directories>] [--trace/-t]
-1>               [--format-assertion/-F]
+1>               [--continue/-c] [--format-assertion/-F]
 1>
 1>        Validate that a schema or a set of schemas are valid with respect
 1>        to their metaschemas.

test/metaschema/fail_directory_continue.clitest

@@ -0,0 +1,35 @@
+MAKE DIRECTORY schemas
+
+WRITE schemas/schema_1.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": "foo"
+}
+EOF
+
+WRITE schemas/schema_2.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": 1
+}
+EOF
+
+// Validation failure
+RUN metaschema schemas --continue --verbose STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+WRITE expected_0.txt UNTIL EOF
+2> fail: schemas/schema_1.json
+2> error: Schema validation failure
+2>   The value was expected to be of type number but it was of type string
+2>     at instance location "/minimum" (line 3, column 3)
+2>     at evaluate path "/properties/minimum/type"
+2>   The object value was expected to validate against the 33 defined properties subschemas
+2>     at instance location "" (line 1, column 1)
+2>     at evaluate path "/properties"
+2> ok: schemas/schema_2.json
+2>   matches http://json-schema.org/draft-04/schema#
+2>
+2> 2 validated, 1 passed, 1 failed
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/metaschema/fail_directory_continue_json.clitest

@@ -0,0 +1,47 @@
+MAKE DIRECTORY schemas
+
+WRITE schemas/schema_1.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": "foo"
+}
+EOF
+
+WRITE schemas/schema_2.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": 1
+}
+EOF
+
+// Validation failure
+RUN metaschema schemas --json --continue STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+WRITE expected_0.txt UNTIL EOF
+1> {
+1>   "valid": false,
+1>   "errors": [
+1>     {
+1>       "keywordLocation": "/properties/minimum/type",
+1>       "absoluteKeywordLocation": "http://json-schema.org/draft-04/schema#/properties/minimum/type",
+1>       "instanceLocation": "/minimum",
+1>       "instancePosition": [ 3, 3, 3, 18 ],
+1>       "error": "The value was expected to be of type number but it was of type string"
+1>     },
+1>     {
+1>       "keywordLocation": "/properties",
+1>       "absoluteKeywordLocation": "http://json-schema.org/draft-04/schema#/properties",
+1>       "instanceLocation": "",
+1>       "instancePosition": [ 1, 1, 4, 1 ],
+1>       "error": "The object value was expected to validate against the 33 defined properties subschemas"
+1>     }
+1>   ]
+1> }
+1> {
+1>   "valid": true
+1> }
+2> schemas/schema_1.json
+2> schemas/schema_2.json
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/metaschema/fail_directory_stop.clitest

@@ -0,0 +1,34 @@
+MAKE DIRECTORY schemas
+
+WRITE schemas/schema_1.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": "foo"
+}
+EOF
+
+WRITE schemas/schema_2.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": 1
+}
+EOF
+
+// Validation failure
+RUN metaschema schemas --verbose STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+WRITE expected_0.txt UNTIL EOF
+2> fail: schemas/schema_1.json
+2> error: Schema validation failure
+2>   The value was expected to be of type number but it was of type string
+2>     at instance location "/minimum" (line 3, column 3)
+2>     at evaluate path "/properties/minimum/type"
+2>   The object value was expected to validate against the 33 defined properties subschemas
+2>     at instance location "" (line 1, column 1)
+2>     at evaluate path "/properties"
+2>
+2> 1 validated, 0 passed, 1 failed
+2> warning: Stopped at first failure, pass --continue/-c to keep going
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/metaschema/fail_directory_stop_json.clitest

@@ -0,0 +1,44 @@
+MAKE DIRECTORY schemas
+
+WRITE schemas/schema_1.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": "foo"
+}
+EOF
+
+WRITE schemas/schema_2.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": 1
+}
+EOF
+
+// Validation failure
+RUN metaschema schemas --json STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+WRITE expected_0.txt UNTIL EOF
+1> {
+1>   "valid": false,
+1>   "errors": [
+1>     {
+1>       "keywordLocation": "/properties/minimum/type",
+1>       "absoluteKeywordLocation": "http://json-schema.org/draft-04/schema#/properties/minimum/type",
+1>       "instanceLocation": "/minimum",
+1>       "instancePosition": [ 3, 3, 3, 18 ],
+1>       "error": "The value was expected to be of type number but it was of type string"
+1>     },
+1>     {
+1>       "keywordLocation": "/properties",
+1>       "absoluteKeywordLocation": "http://json-schema.org/draft-04/schema#/properties",
+1>       "instanceLocation": "",
+1>       "instancePosition": [ 1, 1, 4, 1 ],
+1>       "error": "The object value was expected to validate against the 33 defined properties subschemas"
+1>     }
+1>   ]
+1> }
+2> schemas/schema_1.json
+2> warning: Stopped at first failure, pass --continue/-c to keep going
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt