Skip to content

qms

QMSTest

Bases: Trace

Automate QMS testing and report generation.

Parameters:

Name Type Description Default
test_descriptions Union[str, List[str]]

List of text-based descriptions.

 required
test_criterias Union[List[Callable], Callable]

List of test functions. Function input argument names needs to match keys from the data dictionary.

 required
test_title str

Title of the test.

 'QMSTest'
json_output str

Path into which to write the output results JSON.

 ''
doc_output str

Path into which to write the output QMS summary report (docx).

 ''

Raises:

Type Description
AssertionError

If the number of test_descriptions and test_criteria do not match.
Source code in fastestimator\fastestimator\trace\io\qms.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
class QMSTest(Trace):
    """Automate QMS testing and report generation.

    Args:
        test_descriptions: List of text-based descriptions.
        test_criterias: List of test functions. Function input argument names needs to match keys from the data
            dictionary.
        test_title: Title of the test.
        json_output: Path into which to write the output results JSON.
        doc_output: Path into which to write the output QMS summary report (docx).

    Raises:
        AssertionError: If the number of `test_descriptions` and `test_criteria` do not match.
    """
    def __init__(self,
                 test_descriptions: Union[str, List[str]],
                 test_criterias: Union[List[Callable], Callable],
                 test_title: str = "QMSTest",
                 json_output: str = "",
                 doc_output: str = "") -> None:

        self.json_output = json_output
        self.doc_output = doc_output
        self.test_title = test_title
        self.test_descriptions = to_list(test_descriptions)
        self.test_criterias = to_list(test_criterias)
        assert len(self.test_descriptions) == len(self.test_criterias), "inconsistent input length found"
        all_inputs = set()
        for criteria in self.test_criterias:
            all_inputs.update(inspect.signature(criteria).parameters.keys())
        super().__init__(inputs=all_inputs, mode="test")
        self.total_pass, self.total_fail = 0, 0

    def _initialize_json_summary(self) -> None:
        """Initialize json summary
        """
        self.json_summary = {"title": self.test_title, "stories": []}

    def on_begin(self, data: Data) -> None:
        self._initialize_json_summary()
        self.total_pass, self.total_fail = 0, 0

    def on_epoch_end(self, data: Data) -> None:
        for criteria, description in zip(self.test_criterias, self.test_descriptions):
            story = {"description": description}
            is_passed = criteria(*[data[var_name] for var_name in list(inspect.signature(criteria).parameters.keys())])
            story["passed"] = str(is_passed)

            if is_passed:
                self.total_pass += 1
            else:
                self.total_fail += 1

            self.json_summary["stories"].append(story)

    def on_end(self, data: Data) -> None:
        if self.json_output.endswith(".json"):
            json_path = self.json_output
        else:
            json_path = os.path.join(self.json_output, "QMS.json")
        with open(json_path, 'w') as fp:
            json.dump(self.json_summary, fp, indent=4)
        print("Saved QMS JSON report to {}".format(json_path))

        if self.doc_output.endswith(".docx"):
            doc_path = self.doc_output
        else:
            doc_path = os.path.join(self.doc_output, "QMS_summary.docx")

        doc_summary = _QMSDocx(self.total_pass, self.total_fail)
        doc_summary.save(doc_path)
        print("Saved QMS summary report to {}".format(doc_path))