mlrl.testbed package

This module provides a command line utility for evaluating machine learning algorithms.

Submodules

• mlrl.testbed.characteristics module
  • Characteristic
    • Characteristic.format()
  • OutputCharacteristics
    • OutputCharacteristics.avg_label_cardinality
    • OutputCharacteristics.avg_label_imbalance_ratio
    • OutputCharacteristics.format()
    • OutputCharacteristics.num_distinct_label_vectors
    • OutputCharacteristics.output_density
    • OutputCharacteristics.output_sparsity
    • OutputCharacteristics.tabularize()
  • density()
  • distinct_label_vectors()
  • label_cardinality()
  • label_imbalance_ratio()
• mlrl.testbed.data module
  • Feature
  • FeatureType
    • FeatureType.NOMINAL
    • FeatureType.NUMERICAL
    • FeatureType.ORDINAL
  • MetaData
    • MetaData.get_feature_indices()
    • MetaData.get_num_features()
  • Output
  • load_data_set()
  • load_data_set_and_meta_data()
  • one_hot_encode()
  • save_arff_file()
  • save_data_set()
  • save_data_set_and_meta_data()
  • save_meta_data()
• mlrl.testbed.data_characteristics module
  • DataCharacteristicsWriter
    • DataCharacteristicsWriter.CsvSink
    • DataCharacteristicsWriter.DataCharacteristics
      • DataCharacteristicsWriter.DataCharacteristics.format()
      • DataCharacteristicsWriter.DataCharacteristics.tabularize()
    • DataCharacteristicsWriter.LogSink
  • FeatureCharacteristics
    • FeatureCharacteristics.feature_density
    • FeatureCharacteristics.feature_sparsity
    • FeatureCharacteristics.num_features
    • FeatureCharacteristics.num_nominal_features
    • FeatureCharacteristics.num_numerical_features
    • FeatureCharacteristics.num_ordinal_features
• mlrl.testbed.data_splitting module
  • CrossValidationFold
    • CrossValidationFold.get_fold()
    • CrossValidationFold.get_num_folds()
    • CrossValidationFold.is_last_fold()
    • CrossValidationFold.is_train_test_separated()
  • CrossValidationOverall
    • CrossValidationOverall.get_fold()
    • CrossValidationOverall.get_num_folds()
    • CrossValidationOverall.is_last_fold()
    • CrossValidationOverall.is_train_test_separated()
  • CrossValidationSplitter
  • DataSet
  • DataSplit
    • DataSplit.get_fold()
    • DataSplit.get_num_folds()
    • DataSplit.is_cross_validation_used()
    • DataSplit.is_last_fold()
    • DataSplit.is_train_test_separated()
  • DataSplitter
    • DataSplitter.Callback
      • DataSplitter.Callback.train_and_evaluate()
    • DataSplitter.run()
  • DataType
    • DataType.TEST
    • DataType.TRAINING
    • DataType.get_file_name()
  • NoSplit
    • NoSplit.get_fold()
    • NoSplit.get_num_folds()
    • NoSplit.is_last_fold()
    • NoSplit.is_train_test_separated()
  • NoSplitter
  • TrainTestSplitter
  • TrainingTestSplit
    • TrainingTestSplit.get_fold()
    • TrainingTestSplit.get_num_folds()
    • TrainingTestSplit.is_last_fold()
    • TrainingTestSplit.is_train_test_separated()
  • check_if_files_exist()
• mlrl.testbed.evaluation module
  • BinaryEvaluationWriter
  • EvaluationFunction
    • EvaluationFunction.evaluate()
  • EvaluationWriter
    • EvaluationWriter.CsvSink
      • EvaluationWriter.CsvSink.write_output()
    • EvaluationWriter.EvaluationResult
      • EvaluationWriter.EvaluationResult.avg()
      • EvaluationWriter.EvaluationResult.avg_dict()
      • EvaluationWriter.EvaluationResult.dict()
      • EvaluationWriter.EvaluationResult.format()
      • EvaluationWriter.EvaluationResult.get()
      • EvaluationWriter.EvaluationResult.put()
      • EvaluationWriter.EvaluationResult.tabularize()
    • EvaluationWriter.KWARG_FOLD
    • EvaluationWriter.LogSink
      • EvaluationWriter.LogSink.write_output()
  • RankingEvaluationWriter
  • RegressionEvaluationWriter
• mlrl.testbed.experiments module
  • Evaluation
    • Evaluation.predict_and_evaluate()
  • Experiment
    • Experiment.ExecutionHook
      • Experiment.ExecutionHook.execute()
    • Experiment.run()
    • Experiment.train_and_evaluate()
  • GlobalEvaluation
    • GlobalEvaluation.predict_and_evaluate()
  • IncrementalEvaluation
    • IncrementalEvaluation.predict_and_evaluate()
• mlrl.testbed.format module
  • Formatter
    • Formatter.format()
  • filter_formatters()
  • format_array()
  • format_duration()
  • format_float()
  • format_percentage()
  • format_table()
• mlrl.testbed.info module
  • get_package_info()
• mlrl.testbed.io module
  • clear_directory()
  • create_csv_dict_reader()
  • create_csv_dict_writer()
  • get_file_name()
  • get_file_name_per_fold()
  • open_readable_csv_file()
  • open_writable_csv_file()
  • open_writable_txt_file()
  • write_xml_file()
• mlrl.testbed.label_vectors module
  • LabelVectorSetWriter
    • LabelVectorSetWriter.Visitor
      • LabelVectorSetWriter.Visitor.visit_label_vector()
  • LabelVectorWriter
    • LabelVectorWriter.CsvSink
    • LabelVectorWriter.LabelVectors
      • LabelVectorWriter.LabelVectors.COLUMN_FREQUENCY
      • LabelVectorWriter.LabelVectors.COLUMN_INDEX
      • LabelVectorWriter.LabelVectors.COLUMN_LABEL_VECTOR
      • LabelVectorWriter.LabelVectors.format()
      • LabelVectorWriter.LabelVectors.tabularize()
    • LabelVectorWriter.LogSink
• mlrl.testbed.main module
  • main()
• mlrl.testbed.model_characteristics module
  • ModelCharacteristicsWriter
    • ModelCharacteristicsWriter.CsvSink
    • ModelCharacteristicsWriter.LogSink
  • RuleModelCharacteristicsWriter
    • RuleModelCharacteristicsWriter.RuleModelCharacteristics
      • RuleModelCharacteristicsWriter.RuleModelCharacteristics.format()
      • RuleModelCharacteristicsWriter.RuleModelCharacteristics.tabularize()
    • RuleModelCharacteristicsWriter.RuleModelCharacteristicsVisitor
      • RuleModelCharacteristicsWriter.RuleModelCharacteristicsVisitor.visit_complete_head()
      • RuleModelCharacteristicsWriter.RuleModelCharacteristicsVisitor.visit_conjunctive_body()
      • RuleModelCharacteristicsWriter.RuleModelCharacteristicsVisitor.visit_empty_body()
      • RuleModelCharacteristicsWriter.RuleModelCharacteristicsVisitor.visit_partial_head()
• mlrl.testbed.models module
  • ModelWriter
    • ModelWriter.LogSink
    • ModelWriter.TxtSink
  • RuleModelWriter
    • RuleModelWriter.RuleModelFormattable
      • RuleModelWriter.RuleModelFormattable.format()
      • RuleModelWriter.RuleModelFormattable.visit_complete_head()
      • RuleModelWriter.RuleModelFormattable.visit_conjunctive_body()
      • RuleModelWriter.RuleModelFormattable.visit_empty_body()
      • RuleModelWriter.RuleModelFormattable.visit_partial_head()
• mlrl.testbed.output_writer module
  • Formattable
    • Formattable.format()
  • OutputWriter
    • OutputWriter.CsvSink
      • OutputWriter.CsvSink.write_output()
    • OutputWriter.KWARG_DATA_SPLIT
    • OutputWriter.LogSink
      • OutputWriter.LogSink.write_output()
    • OutputWriter.Sink
      • OutputWriter.Sink.write_output()
    • OutputWriter.TxtSink
      • OutputWriter.TxtSink.write_output()
    • OutputWriter.write_output()
  • Tabularizable
    • Tabularizable.tabularize()
• mlrl.testbed.parameters module
  • ParameterCsvInput
    • ParameterCsvInput.read_parameters()
  • ParameterInput
    • ParameterInput.read_parameters()
  • ParameterWriter
    • ParameterWriter.CsvSink
    • ParameterWriter.LogSink
    • ParameterWriter.Parameters
      • ParameterWriter.Parameters.format()
      • ParameterWriter.Parameters.tabularize()
• mlrl.testbed.persistence module
  • ModelPersistence
    • ModelPersistence.load_model()
    • ModelPersistence.save_model()
• mlrl.testbed.prediction_characteristics module
  • PredictionCharacteristicsWriter
    • PredictionCharacteristicsWriter.CsvSink
    • PredictionCharacteristicsWriter.LogSink
• mlrl.testbed.prediction_scope module
  • GlobalPrediction
    • GlobalPrediction.get_file_name()
    • GlobalPrediction.get_model_size()
    • GlobalPrediction.is_global()
  • IncrementalPrediction
    • IncrementalPrediction.get_file_name()
    • IncrementalPrediction.get_model_size()
    • IncrementalPrediction.is_global()
  • PredictionScope
    • PredictionScope.get_file_name()
    • PredictionScope.get_model_size()
    • PredictionScope.is_global()
  • PredictionType
    • PredictionType.BINARY
    • PredictionType.PROBABILITIES
    • PredictionType.SCORES
    • PredictionType.parse()
• mlrl.testbed.predictions module
  • PredictionWriter
    • PredictionWriter.ArffSink
      • PredictionWriter.ArffSink.write_output()
    • PredictionWriter.LogSink
    • PredictionWriter.Predictions
      • PredictionWriter.Predictions.format()
• mlrl.testbed.probability_calibration module
  • JointProbabilityCalibrationModelWriter
    • JointProbabilityCalibrationModelWriter.CsvSink
    • JointProbabilityCalibrationModelWriter.LogSink
  • MarginalProbabilityCalibrationModelWriter
    • MarginalProbabilityCalibrationModelWriter.CsvSink
    • MarginalProbabilityCalibrationModelWriter.LogSink
  • ProbabilityCalibrationModelWriter
    • ProbabilityCalibrationModelWriter.IsotonicProbabilityCalibrationModelFormattable
      • ProbabilityCalibrationModelWriter.IsotonicProbabilityCalibrationModelFormattable.format()
      • ProbabilityCalibrationModelWriter.IsotonicProbabilityCalibrationModelFormattable.tabularize()
      • ProbabilityCalibrationModelWriter.IsotonicProbabilityCalibrationModelFormattable.visit_bin()
    • ProbabilityCalibrationModelWriter.NoProbabilityCalibrationModelFormattable
      • ProbabilityCalibrationModelWriter.NoProbabilityCalibrationModelFormattable.format()
      • ProbabilityCalibrationModelWriter.NoProbabilityCalibrationModelFormattable.tabularize()
• mlrl.testbed.problem_type module
  • ProblemType
    • ProblemType.CLASSIFICATION
    • ProblemType.REGRESSION
    • ProblemType.parse()
• mlrl.testbed.runnables module
  • LearnerRunnable
    • LearnerRunnable.ClearOutputDirHook
      • LearnerRunnable.ClearOutputDirHook.execute()
    • LearnerRunnable.DATA_SPLIT_CROSS_VALIDATION
    • LearnerRunnable.DATA_SPLIT_TRAIN_TEST
    • LearnerRunnable.DATA_SPLIT_VALUES
    • LearnerRunnable.OPTION_CURRENT_FOLD
    • LearnerRunnable.OPTION_NUM_FOLDS
    • LearnerRunnable.OPTION_TEST_SIZE
    • LearnerRunnable.PARAM_DATA_SPLIT
    • LearnerRunnable.PARAM_OUTPUT_DIR
    • LearnerRunnable.PARAM_PREDICTION_TYPE
    • LearnerRunnable.PARAM_PRINT_DATA_CHARACTERISTICS
    • LearnerRunnable.PARAM_PRINT_EVALUATION
    • LearnerRunnable.PARAM_PRINT_LABEL_VECTORS
    • LearnerRunnable.PARAM_PRINT_PREDICTIONS
    • LearnerRunnable.PARAM_PRINT_PREDICTION_CHARACTERISTICS
    • LearnerRunnable.PARAM_PROBLEM_TYPE
    • LearnerRunnable.PARAM_RANDOM_STATE
    • LearnerRunnable.PARAM_STORE_DATA_CHARACTERISTICS
    • LearnerRunnable.PARAM_STORE_EVALUATION
    • LearnerRunnable.PARAM_STORE_LABEL_VECTORS
    • LearnerRunnable.PARAM_STORE_PREDICTIONS
    • LearnerRunnable.PARAM_STORE_PREDICTION_CHARACTERISTICS
    • LearnerRunnable.PRINT_DATA_CHARACTERISTICS_VALUES
    • LearnerRunnable.PRINT_EVALUATION_VALUES
    • LearnerRunnable.PRINT_LABEL_VECTORS_VALUES
    • LearnerRunnable.PRINT_PREDICTIONS_VALUES
    • LearnerRunnable.PRINT_PREDICTION_CHARACTERISTICS_VALUES
    • LearnerRunnable.STORE_DATA_CHARACTERISTICS_VALUES
    • LearnerRunnable.STORE_EVALUATION_VALUES
    • LearnerRunnable.STORE_LABEL_VECTORS_VALUES
    • LearnerRunnable.STORE_PREDICTIONS_VALUES
    • LearnerRunnable.STORE_PREDICTION_CHARACTERISTICS_VALUES
    • LearnerRunnable.configure_arguments()
    • LearnerRunnable.configure_problem_specific_arguments()
    • LearnerRunnable.create_classifier()
    • LearnerRunnable.create_regressor()
  • LogLevel
    • LogLevel.CRITICAL
    • LogLevel.DEBUG
    • LogLevel.ERROR
    • LogLevel.FATAL
    • LogLevel.INFO
    • LogLevel.NOTSET
    • LogLevel.WARN
    • LogLevel.WARNING
    • LogLevel.parse()
  • RuleLearnerRunnable
    • RuleLearnerRunnable.INCREMENTAL_EVALUATION_VALUES
    • RuleLearnerRunnable.OPTION_MAX_SIZE
    • RuleLearnerRunnable.OPTION_MIN_SIZE
    • RuleLearnerRunnable.OPTION_STEP_SIZE
    • RuleLearnerRunnable.PARAM_FEATURE_FORMAT
    • RuleLearnerRunnable.PARAM_INCREMENTAL_EVALUATION
    • RuleLearnerRunnable.PARAM_PRINT_JOINT_PROBABILITY_CALIBRATION_MODEL
    • RuleLearnerRunnable.PARAM_PRINT_MARGINAL_PROBABILITY_CALIBRATION_MODEL
    • RuleLearnerRunnable.PARAM_PRINT_RULES
    • RuleLearnerRunnable.PARAM_SPARSE_FEATURE_VALUE
    • RuleLearnerRunnable.PARAM_STORE_JOINT_PROBABILITY_CALIBRATION_MODEL
    • RuleLearnerRunnable.PARAM_STORE_MARGINAL_PROBABILITY_CALIBRATION_MODEL
    • RuleLearnerRunnable.PARAM_STORE_RULES
    • RuleLearnerRunnable.PRINT_JOINT_PROBABILITY_CALIBRATION_MODEL_VALUES
    • RuleLearnerRunnable.PRINT_MARGINAL_PROBABILITY_CALIBRATION_MODEL_VALUES
    • RuleLearnerRunnable.PRINT_RULES_VALUES
    • RuleLearnerRunnable.STORE_JOINT_PROBABILITY_CALIBRATION_MODEL_VALUES
    • RuleLearnerRunnable.STORE_MARGINAL_PROBABILITY_CALIBRATION_MODEL_VALUES
    • RuleLearnerRunnable.STORE_RULES_VALUES
    • RuleLearnerRunnable.configure_problem_specific_arguments()
    • RuleLearnerRunnable.create_classifier()
    • RuleLearnerRunnable.create_regressor()
  • Runnable
    • Runnable.ProgramInfo
      • Runnable.ProgramInfo.all_python_packages
      • Runnable.ProgramInfo.authors
      • Runnable.ProgramInfo.name
      • Runnable.ProgramInfo.python_packages
      • Runnable.ProgramInfo.version
      • Runnable.ProgramInfo.year
    • Runnable.configure_arguments()
    • Runnable.configure_logger()
    • Runnable.get_program_info()
    • Runnable.run()